From fa73fb30d0a79e8c6b0c392e42bc80f880fd3b33 Mon Sep 17 00:00:00 2001 From: nroope Date: Thu, 26 Mar 2026 10:46:09 +0100 Subject: [PATCH 01/22] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4dd443f..c6e0c9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ optional-dependencies.all = [ "pytest>=8.4", "tensorflow>=2.17,<=2.20", "torch>= optional-dependencies.tensorflow = [ "tensorflow>=2.17,<=2.20" ] optional-dependencies.test = [ "pytest>=8.4" ] optional-dependencies.torch = [ "torch>=2.1" ] -urls.repository = "https://github.com/nroope/PQuant" +urls.repository = "https://github.com/nroope/PQuantML" [tool.setuptools] packages = [ "pquant" ] From 63d425899f2b18c8b318142fabb08806025c7070 Mon Sep 17 00:00:00 2001 From: nroope Date: Thu, 26 Mar 2026 11:01:52 +0100 Subject: [PATCH 02/22] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c6e0c9e..fb0c210 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ optional-dependencies.all = [ "pytest>=8.4", "tensorflow>=2.17,<=2.20", "torch>= optional-dependencies.tensorflow = [ "tensorflow>=2.17,<=2.20" ] optional-dependencies.test = [ "pytest>=8.4" ] optional-dependencies.torch = [ "torch>=2.1" ] -urls.repository = "https://github.com/nroope/PQuantML" +urls.repository = "https://github.com/cern-nextgen/PQuantML" [tool.setuptools] packages = [ "pquant" ] From 44e268c62eecc4330cb19e5233b19d32d8e684ea Mon Sep 17 00:00:00 2001 From: nroope Date: Thu, 26 Mar 2026 11:12:51 +0100 Subject: [PATCH 03/22] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d5d245b..eda8049 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ The various pruning methods have different training steps, such as a pre-trainin ### Example -Example notebook can be found [here](https://github.com/nroope/PQuant/tree/main/examples). It handles the +Example notebook can be found [here](https://github.com/cern-nextgen/PQuantML/tree/main/examples). It handles the 1. Creation of a torch model and data loaders. 2. Creation of the training and validation functions. 3. Loading a default pruning configuration of a pruning method. From 7f22ad59f2717fbe2db35a576f41fc0ef00a7092 Mon Sep 17 00:00:00 2001 From: Anastasiia Petrovych Date: Wed, 22 Apr 2026 11:30:18 +0200 Subject: [PATCH 04/22] Add backend addapter for hpo platform and fixed serialization issue in pdp method (#31) --- .readthedocs.yaml | 1 - README.md | 8 +- docs/Makefile | 2 +- docs/requirements.txt | 4 +- docs/source/_static/custom.css | 48 ++++----- docs/source/conf.py | 2 +- docs/source/faq.md | 4 +- docs/source/getting_started.md | 30 +++--- docs/source/index.rst | 10 +- docs/source/install.md | 2 +- docs/source/status.md | 4 +- .../core/hyperparameter_optimization.py | 102 ++++++++++++++---- src/pquant/data_models/quantization_model.py | 2 +- src/pquant/data_models/training_model.py | 2 - 14 files changed, 136 insertions(+), 85 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 575f578..f72c324 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -20,4 +20,3 @@ sphinx: # python: # install: # - requirements: docs/requirements.txt - diff --git a/README.md b/README.md index eda8049..f10cb68 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,14 @@ ## Prune and Quantize ML models PQuant is a library for training compressed machine learning models, developed at CERN as part of the [Next Generation Triggers](https://nextgentriggers.web.cern.ch/t13/) project. -Installation via pip: ```pip install pquant-ml```. +Installation via pip: ```pip install pquant-ml```. -With TensorFlow ```pip install pquant-ml[tensorflow]```. +With TensorFlow ```pip install pquant-ml[tensorflow]```. With PyTorch ```pip install pquant-ml[torch]```. -PQuant replaces the layers and activations it finds with a Compressed (in the case of layers) or Quantized (in the case of activations) variant. These automatically handle the quantization of the weights, biases and activations, and the pruning of the weights. -Both PyTorch and TensorFlow models are supported. +PQuant replaces the layers and activations it finds with a Compressed (in the case of layers) or Quantized (in the case of activations) variant. These automatically handle the quantization of the weights, biases and activations, and the pruning of the weights. +Both PyTorch and TensorFlow models are supported. ### Layers that can be compressed diff --git a/docs/Makefile b/docs/Makefile index 5647f38..e88d665 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -18,4 +18,4 @@ help: # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @sphinx-apidoc -f -o autodoc/ ../src/HGQ - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/requirements.txt b/docs/requirements.txt index 726ada1..950c059 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ -sphinx furo myst-parser -sphinx_rtd_theme +sphinx sphinx-autodoc-typehints +sphinx_rtd_theme diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index 3300cdf..e9e710a 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -51,12 +51,12 @@ html body nav.wy-nav-top, } .wy-body-for-nav { - background-color: #ffffff !important; + background-color: #ffffff !important; } .wy-nav-content { background-color: #ffffff !important; - max-width: 1200px !important; + max-width: 1200px !important; } .wy-side-nav-search { @@ -71,19 +71,19 @@ html body nav.wy-nav-top, } .wy-nav-side { - background-color: #b30000 !important; + background-color: #b30000 !important; } .wy-menu-vertical a { - color: #ffffff !important; + color: #ffffff !important; } .wy-menu-vertical a:hover { - background-color: #990000 !important; + background-color: #990000 !important; } .wy-menu-vertical li.current > a, .wy-menu-vertical li.toctree-l1.current > a { - background-color: #660000 !important; + background-color: #660000 !important; color: #ffffff !important; } @@ -110,7 +110,7 @@ h1, h2, h3, h4, h5, h6 { background-color: #ffffff !important; } -.rst-content > .document > .toctree-wrapper, +.rst-content > .document > .toctree-wrapper, .rst-content > .document > .section { background-color: #ffffff !important; } @@ -121,7 +121,7 @@ h1, h2, h3, h4, h5, h6 { } .rst-content table th { - background-color: #ffe6e6 !important; + background-color: #ffe6e6 !important; color: #b30000 !important; } @@ -158,7 +158,7 @@ h1, h2, h3, h4, h5, h6 { color: #000000 !important; padding: 8px !important; margin: 12px !important; - width: calc(100% - 24px) !important; + width: calc(100% - 24px) !important; box-sizing: border-box !important; } @@ -170,7 +170,7 @@ h1, h2, h3, h4, h5, h6 { .wy-side-nav-search { background-color: #ffffff !important; - padding: 20px 15px !important; + padding: 20px 15px !important; border-bottom: 2px solid #b30000 !important; } @@ -184,19 +184,19 @@ h1, h2, h3, h4, h5, h6 { } .wy-side-nav-search .version-switch { - color: #b30000 !important; + color: #b30000 !important; font-weight: 600 !important; } .wy-side-nav-search .version-switch :hover { - color: #990000 !important; + color: #990000 !important; } .wy-side-nav-search .fa-caret-down { color: #b30000 !important; } .wy-side-nav-search select.version-switch { - color: #b30000 !important; + color: #b30000 !important; background-color: #ffffff !important; border: 2px solid #b30000 !important; font-weight: 600 !important; @@ -205,8 +205,8 @@ h1, h2, h3, h4, h5, h6 { } .wy-side-nav-search select.rtd-version-select { - color: #b30000 !important; - background-color: #ffffff !important; + color: #b30000 !important; + background-color: #ffffff !important; border: 2px solid #b30000 !important; font-weight: 600 !important; padding: 6px !important; @@ -223,16 +223,16 @@ h1, h2, h3, h4, h5, h6 { } .wy-side-nav-search input[type="search"] { - height: 100% !important; - width: 100% !important; - font-size: 18px !important; - padding: 10px 14px !important; + height: 100% !important; + width: 100% !important; + font-size: 18px !important; + padding: 10px 14px !important; margin: 0 !important; - border-radius: 6px !important; - border: none !important; - box-shadow: none !important; - background: #ffffff !important; - color: #000000 !important; + border-radius: 6px !important; + border: none !important; + box-shadow: none !important; + background: #ffffff !important; + color: #000000 !important; box-sizing: border-box !important; } diff --git a/docs/source/conf.py b/docs/source/conf.py index 64ac78d..fe398a4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -67,4 +67,4 @@ html_css_files = [ 'custom.css', -] \ No newline at end of file +] diff --git a/docs/source/faq.md b/docs/source/faq.md index 5a3f9f1..c8637a9 100644 --- a/docs/source/faq.md +++ b/docs/source/faq.md @@ -12,12 +12,12 @@ An example to install PyTorch with CUDA 13.0: pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cu130 ``` ## Can I use MLflow locally? -Yes. +Yes. PQuantML integrates with MLflow for experiment tracking and model logging and local usage is fully supported. -### Start local MLFlow UI: +### Start local MLFlow UI: ```python mlflow ui --host 0.0.0.0 --port 5000 ``` diff --git a/docs/source/getting_started.md b/docs/source/getting_started.md index 63b5f0b..c5c99d6 100644 --- a/docs/source/getting_started.md +++ b/docs/source/getting_started.md @@ -39,13 +39,13 @@ def build_model(config): class Model(torch.nn.Module): def __init__(self): super().__init__() - self.dense1 = PQDense(config, 16, 64, + self.dense1 = PQDense(config, 16, 64, in_quant_bits = (1, 3, 3)) self.relu = PQActivation(config, "relu") self.dense2 = PQDense(config, 64, 32) self.dense3 = PQDense(config, 32, 32) - self.dense4 = PQDense(config, 32, 5, - quantize_output=True, + self.dense4 = PQDense(config, 32, 5, + quantize_output=True, out_quant_bits=(1, 3, 3)) def forward(self, x): @@ -78,7 +78,7 @@ def build_model(): x = self.relu(self.dense3(x)) x = self.dense4(x) return x - + return Model() @@ -86,10 +86,10 @@ def build_model(): model = add_compression_layers(model, config) ``` -### Fine-Tuning with PQuantML +### Fine-Tuning with PQuantML PQuantML provides an automated fine-tuning and hyperparameter-optimization workflow through the `TuningTask API`. This allows you to search for optimal pruning, quantization, and training parameters using your own training, validation, and objective functions. -```python +```python from pquant.core.finetuning import TuningTask, TuningConfig # Convert defined yaml file into the object @@ -142,13 +142,13 @@ Training is handled through the `train_model(...)` wrapper: ```python from pquant import train_model -trained_model = train_model(model = model, - config = config, - train_func = ..., - valid_func = ..., - trainloader = ..., +trained_model = train_model(model = model, + config = config, + train_func = ..., + valid_func = ..., + trainloader = ..., device="cuda", - testloader = ..., + testloader = ..., loss_func = loss_func, optimizer = optimizer, scheduler=scheduler @@ -164,15 +164,15 @@ def build_model(config): class Model(torch.nn.Module): def __init__(self): super().__init__() - self.dense1 = PQDense(config, 16, 64, + self.dense1 = PQDense(config, 16, 64, in_quant_bits = (1, 3, 3)) self.relu1 = PQActivation(config, "relu") self.relu2 = PQActivation(config, "relu") self.relu3 = PQActivation(config, "relu") self.dense2 = PQDense(config, 64, 32) self.dense3 = PQDense(config, 32, 32) - self.dense4 = PQDense(config, 32, 5, - quantize_output=True, + self.dense4 = PQDense(config, 32, 5, + quantize_output=True, out_quant_bits=(1, 3, 3)) def forward(self, x): diff --git a/docs/source/index.rst b/docs/source/index.rst index 344ba5d..e697789 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -16,10 +16,10 @@ PQuantML Welcome to the official documentation for **PQuantML**, a hardware-aware model compression framework supporting: -- Joint pruning + quantization -- Layer-wise precision configuration -- Flexible training pipelines -- PyTorch and TensorFlow backends +- Joint pruning + quantization +- Layer-wise precision configuration +- Flexible training pipelines +- PyTorch and TensorFlow backends - Integration with hardware-friendly toolchains (e.g., hls4ml) PQuantML enables efficient deployment of compact neural networks on resource-constrained hardware such as FPGAs and embedded accelerators. @@ -49,7 +49,7 @@ Contents .. toctree:: :maxdepth: 2 - + status install getting_started diff --git a/docs/source/install.md b/docs/source/install.md index ef039ab..e9ba183 100644 --- a/docs/source/install.md +++ b/docs/source/install.md @@ -1,6 +1,6 @@ # Installation -Use `pip install pquant-ml` to install the latest version from PyPI. You will need an environment with `python>=3.10,<=3.12` installed. +Use `pip install pquant-ml` to install the latest version from PyPI. You will need an environment with `python>=3.10,<=3.12` installed. ```{warning} diff --git a/docs/source/status.md b/docs/source/status.md index 15f1aa9..8dd291f 100644 --- a/docs/source/status.md +++ b/docs/source/status.md @@ -1,6 +1,6 @@ # PQuantML Status -This page tracks the development status of PQuantML features +This page tracks the development status of PQuantML features ## Release: v1.0.0 @@ -13,5 +13,3 @@ This page tracks the development status of PQuantML features | hls4ml integration | βœ… Complete | Works in v1.0.0 | | FITCompress | 🚧 Partially implemented | Works through PyTorch only | | Documentation | 🚧 Improving | Expanded daily | - - diff --git a/src/pquant/core/hyperparameter_optimization.py b/src/pquant/core/hyperparameter_optimization.py index 55a6077..85b0560 100644 --- a/src/pquant/core/hyperparameter_optimization.py +++ b/src/pquant/core/hyperparameter_optimization.py @@ -6,7 +6,6 @@ import keras import optuna -import torch import yaml from pydantic import BaseModel, Field, field_validator @@ -37,10 +36,9 @@ def get_sampler(sampler_type, **kwargs): raise ValueError(f"Unknown sampler type: {sampler_type}") -def log_model_by_backend(model, name, signature=None, registered_model_name=None): +def log_model_by_backend(model, name, backend, signature=None, registered_model_name=None): import mlflow - backend = keras.backend.backend() kwargs = { "artifact_path": name, "signature": signature, @@ -61,7 +59,7 @@ class MetricFunction(BaseModel): @field_validator('direction') def validate_direction(cls, direction): if direction not in constants.FINETUNING_DIRECTION: - raise ValueError("direction must be 'maximize' or 'minimize'") + raise ValueError("Direction must be 'maximize' or 'minimize'") return direction @@ -115,6 +113,58 @@ def get_dict(self): return self.model_dump(mode="json") +class BackendAdapter: + def __init__(self, model): + self.backend = self._detect_backend(model) + self.device = None + + def clone_model(self, model): + if self.backend == constants.TORCH_BACKEND: + return copy.deepcopy(model) + elif self.backend == constants.TF_BACKEND: + new_model = keras.models.clone_model(model) + new_model.set_weights(model.get_weights()) + return new_model + + def get_backend(self): + return self.backend + + def get_device(self): + return self.device + + def _detect_backend(self, model): + if hasattr(model, "parameters"): + return constants.TORCH_BACKEND + elif isinstance(model, keras.Model): + return constants.TF_BACKEND + else: + raise ValueError("Unsupported model type") + + def move_to_device(self, model): + if self.backend == constants.TORCH_BACKEND: + self.device = next(model.parameters()).device + return model.to(self.device) + return model + + def eval(self, model): + if self.backend == constants.TORCH_BACKEND: + model.eval() + return model + + def tensor_to_numpy(self, tensor): + if self.backend == constants.TORCH_BACKEND: + return tensor.detach().cpu().numpy() + elif self.backend == constants.TF_BACKEND: + return tensor.numpy() + + def forward(self, model, x): + if self.backend == constants.TORCH_BACKEND: + x = x.to(self.device) + return model(x) + elif self.backend == constants.TF_BACKEND: + return model(x, training=False) + + class TuningTask: def __init__(self, config: PQConfig): self.config = config @@ -124,7 +174,6 @@ def __init__(self, config: PQConfig): self._validation_function: Optional[Callable] = None self._optimizer_function: Optional[Callable] = None self._scheduler_function: Optional[Callable] = None - self.device = "cuda" if torch.cuda.is_available() else "cpu" self.enable_mlflow = False self.tracking_uri = None self.storage_db = None @@ -245,16 +294,17 @@ def register_hyperparameter(self, name, optuna_func, *args, **kwargs): def objective(self, trial, model, train_func, valid_func, **kwargs): from pquant import add_compression_layers, train_model + config_copy = copy.deepcopy(self.config) for param_name, (optuna_func, func_args, func_kwargs) in self.hyperparameters.items(): new_value = optuna_func(trial, *func_args, **func_kwargs) logging.info(f"Suggested {param_name} = {new_value}") applied = False for sub_config in [ - self.config.training_parameters, - self.config.pruning_parameters, - self.config.quantization_parameters, - self.config.fitcompress_parameters, + config_copy.training_parameters, + config_copy.pruning_parameters, + config_copy.quantization_parameters, + config_copy.fitcompress_parameters, ]: if hasattr(sub_config, param_name): setattr(sub_config, param_name, new_value) @@ -266,29 +316,32 @@ def objective(self, trial, model, train_func, valid_func, **kwargs): trainloader = kwargs['trainloader'] raw_input_batch = next(iter(trainloader)) sample_input = raw_input_batch[0] - sample_output = model(sample_input.to(next(model.parameters()).device)) + model_copy = self.adapter.clone_model(model) + model_copy = self.adapter.move_to_device(model_copy) + sample_output = self.adapter.forward(model_copy, sample_input) input_shape = sample_input.shape - compressed_model = add_compression_layers(model, self.config, input_shape) + compressed_model = add_compression_layers(model_copy, config_copy, input_shape) optimizer_func = self.get_optimizer_function() - optimizer = optimizer_func(self.config, compressed_model) + optimizer = optimizer_func(config_copy, compressed_model) scheduler_func = self.get_scheduler_function() - scheduler = scheduler_func(optimizer, self.config) + scheduler = scheduler_func(optimizer, config_copy) + device = self.adapter.get_device() trained_model = train_model( compressed_model, - self.config, + config_copy, train_func, valid_func, optimizer=optimizer, scheduler=scheduler, - device=self.device, + device=device, writer=None, **kwargs, ) - trained_model.eval() + self.adapter.eval(trained_model) objectives = [ - metric_object.function_name(trained_model, device=self.device, **kwargs) + metric_object.function_name(trained_model, device=device, **kwargs) for _, metric_object in self.objectives.items() ] @@ -297,23 +350,27 @@ def objective(self, trial, model, train_func, valid_func, **kwargs): from mlflow.models import infer_signature with mlflow.start_run(nested=True): - mlflow.log_params({param_name: getattr(self.config, param_name) for param_name in self.config.model_fields}) + mlflow.log_params({param_name: getattr(config_copy, param_name) for param_name in config_copy.model_fields}) mlflow.log_metrics({key: val for key, val in zip(self.objectives.keys(), objectives)}) - signature = infer_signature(sample_input.cpu().numpy(), sample_output.detach().cpu().numpy()) + signature = infer_signature( + self.adapter.tensor_to_numpy(sample_input), self.adapter.tensor_to_numpy(sample_output) + ) mlflow.log_text(yaml.safe_dump(self.get_dict()), "config.yaml") - model_name = self.config.hpo_parameters.model_name + model_name = config_copy.hpo_parameters.model_name log_model_by_backend( model=trained_model, name=model_name, signature=signature, registered_model_name=model_name, + backend=self.adapter.get_backend(), ) return objectives if len(objectives) > 1 else objectives[0] def run_optimization(self, model, **kwargs): hpo_parameters = self.config.hpo_parameters + num_trials = hpo_parameters.num_trials if self.enable_mlflow: import mlflow @@ -330,12 +387,11 @@ def run_optimization(self, model, **kwargs): load_if_exists=True, directions=[metric_object.direction for _, metric_object in self.objectives.items()], ) - - num_trials = hpo_parameters.num_trials + self.adapter = BackendAdapter(model) study.optimize( lambda trial: self.objective( trial, - copy.deepcopy(model.cpu()).to(self.device), + model, self.get_training_function(), self.get_validation_function(), **kwargs, diff --git a/src/pquant/data_models/quantization_model.py b/src/pquant/data_models/quantization_model.py index 31cefd8..a3c71b7 100644 --- a/src/pquant/data_models/quantization_model.py +++ b/src/pquant/data_models/quantization_model.py @@ -1,5 +1,5 @@ -from typing import List from enum import Enum + from pydantic import BaseModel, Field diff --git a/src/pquant/data_models/training_model.py b/src/pquant/data_models/training_model.py index f841d70..1619b59 100644 --- a/src/pquant/data_models/training_model.py +++ b/src/pquant/data_models/training_model.py @@ -1,5 +1,3 @@ -from typing import Literal - from pydantic import BaseModel, ConfigDict, Field From d9fe442a6994bbf2285eee5a8f11d985046d0e7d Mon Sep 17 00:00:00 2001 From: Anastasiia Petrovych Date: Thu, 23 Apr 2026 20:11:30 +0200 Subject: [PATCH 05/22] Switched to the higher version of MLflow (#36) --- src/pquant/core/hyperparameter_optimization.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/pquant/core/hyperparameter_optimization.py b/src/pquant/core/hyperparameter_optimization.py index 85b0560..c9d4289 100644 --- a/src/pquant/core/hyperparameter_optimization.py +++ b/src/pquant/core/hyperparameter_optimization.py @@ -40,7 +40,7 @@ def log_model_by_backend(model, name, backend, signature=None, registered_model_ import mlflow kwargs = { - "artifact_path": name, + "name": name, "signature": signature, "registered_model_name": registered_model_name, } @@ -249,7 +249,7 @@ def set_hyperparameters(self): if numerical_params: self.set_numerical_params(numerical_params) - elif categorical_params: + if categorical_params: self.set_categorical_params(categorical_params) def set_numerical_params(self, numerical_params): @@ -293,8 +293,9 @@ def register_hyperparameter(self, name, optuna_func, *args, **kwargs): def objective(self, trial, model, train_func, valid_func, **kwargs): from pquant import add_compression_layers, train_model - + config_copy = copy.deepcopy(self.config) + applied_parameters = {} for param_name, (optuna_func, func_args, func_kwargs) in self.hyperparameters.items(): new_value = optuna_func(trial, *func_args, **func_kwargs) logging.info(f"Suggested {param_name} = {new_value}") @@ -308,6 +309,7 @@ def objective(self, trial, model, train_func, valid_func, **kwargs): ]: if hasattr(sub_config, param_name): setattr(sub_config, param_name, new_value) + applied_parameters[param_name] = new_value applied = True break if not applied: @@ -315,12 +317,13 @@ def objective(self, trial, model, train_func, valid_func, **kwargs): trainloader = kwargs['trainloader'] raw_input_batch = next(iter(trainloader)) + sample_input = raw_input_batch[0] model_copy = self.adapter.clone_model(model) model_copy = self.adapter.move_to_device(model_copy) sample_output = self.adapter.forward(model_copy, sample_input) - input_shape = sample_input.shape + compressed_model = add_compression_layers(model_copy, config_copy, input_shape) optimizer_func = self.get_optimizer_function() optimizer = optimizer_func(config_copy, compressed_model) @@ -350,7 +353,7 @@ def objective(self, trial, model, train_func, valid_func, **kwargs): from mlflow.models import infer_signature with mlflow.start_run(nested=True): - mlflow.log_params({param_name: getattr(config_copy, param_name) for param_name in config_copy.model_fields}) + mlflow.log_params(applied_parameters) mlflow.log_metrics({key: val for key, val in zip(self.objectives.keys(), objectives)}) signature = infer_signature( self.adapter.tensor_to_numpy(sample_input), self.adapter.tensor_to_numpy(sample_output) From 0e856a0ede49e7b40f208ff7062e26b53b37996e Mon Sep 17 00:00:00 2001 From: nroope Date: Mon, 8 Jun 2026 10:30:51 +0200 Subject: [PATCH 06/22] initial Torch HGQ and pruning layers (#39) initial Torch HGQ and pruning layers * Torch versions of HGQ Quantizer and pruning methods, used by the Torch PQlayers --- src/pquant/__init__.py | 8 +- src/pquant/core/constants.py | 21 - src/pquant/core/keras/layers.py | 2 +- .../keras}/pruning_methods/__init__.py | 0 .../pruning_methods/activation_pruning.py | 0 .../keras}/pruning_methods/autosparse.py | 4 +- .../pruning_methods/constraint_functions.py | 0 .../{ => core/keras}/pruning_methods/cs.py | 0 .../{ => core/keras}/pruning_methods/dst.py | 0 .../{ => core/keras}/pruning_methods/mdmm.py | 39 +- .../pruning_methods/metric_functions.py | 0 .../{ => core/keras}/pruning_methods/pdp.py | 0 .../{ => core/keras}/pruning_methods/wanda.py | 0 src/pquant/core/keras/quantizer.py | 36 +- src/pquant/core/keras/utils.py | 25 + src/pquant/core/quantizer_functions.py | 48 -- src/pquant/core/torch/hgq_quantizer.py | 408 ++++++++++++ src/pquant/core/torch/layers.py | 7 +- .../core/torch/pruning_methods/__init__.py | 0 .../pruning_methods/activation_pruning.py | 104 +++ .../core/torch/pruning_methods/autosparse.py | 143 ++++ .../pruning_methods/constraint_functions.py | 99 +++ src/pquant/core/torch/pruning_methods/cs.py | 79 +++ src/pquant/core/torch/pruning_methods/dst.py | 105 +++ .../core/torch/pruning_methods/fitcompress.py | 45 ++ src/pquant/core/torch/pruning_methods/mdmm.py | 141 ++++ .../torch/pruning_methods/metric_functions.py | 57 ++ src/pquant/core/torch/pruning_methods/pdp.py | 140 ++++ .../core/torch/pruning_methods/wanda.py | 160 +++++ src/pquant/core/torch/quantizer.py | 42 +- src/pquant/core/torch/utils.py | 25 + src/pquant/core/utils.py | 30 - src/pquant/pruning_methods/fitcompress.py | 52 -- tests/test_ap.py | 2 +- tests/test_hgq_torch.py | 250 +++++++ tests/test_pdp.py | 2 +- tests/test_torch_compression_layers.py | 30 +- tests/test_torch_pruning_layers.py | 626 ++++++++++++++++++ tests/test_wanda.py | 2 +- 39 files changed, 2533 insertions(+), 199 deletions(-) rename src/pquant/{ => core/keras}/pruning_methods/__init__.py (100%) rename src/pquant/{ => core/keras}/pruning_methods/activation_pruning.py (100%) rename src/pquant/{ => core/keras}/pruning_methods/autosparse.py (97%) rename src/pquant/{ => core/keras}/pruning_methods/constraint_functions.py (100%) rename src/pquant/{ => core/keras}/pruning_methods/cs.py (100%) rename src/pquant/{ => core/keras}/pruning_methods/dst.py (100%) rename src/pquant/{ => core/keras}/pruning_methods/mdmm.py (77%) rename src/pquant/{ => core/keras}/pruning_methods/metric_functions.py (100%) rename src/pquant/{ => core/keras}/pruning_methods/pdp.py (100%) rename src/pquant/{ => core/keras}/pruning_methods/wanda.py (100%) create mode 100644 src/pquant/core/keras/utils.py delete mode 100644 src/pquant/core/quantizer_functions.py create mode 100644 src/pquant/core/torch/hgq_quantizer.py create mode 100644 src/pquant/core/torch/pruning_methods/__init__.py create mode 100644 src/pquant/core/torch/pruning_methods/activation_pruning.py create mode 100644 src/pquant/core/torch/pruning_methods/autosparse.py create mode 100644 src/pquant/core/torch/pruning_methods/constraint_functions.py create mode 100644 src/pquant/core/torch/pruning_methods/cs.py create mode 100644 src/pquant/core/torch/pruning_methods/dst.py create mode 100644 src/pquant/core/torch/pruning_methods/fitcompress.py create mode 100644 src/pquant/core/torch/pruning_methods/mdmm.py create mode 100644 src/pquant/core/torch/pruning_methods/metric_functions.py create mode 100644 src/pquant/core/torch/pruning_methods/pdp.py create mode 100644 src/pquant/core/torch/pruning_methods/wanda.py create mode 100644 src/pquant/core/torch/utils.py delete mode 100644 src/pquant/pruning_methods/fitcompress.py create mode 100644 tests/test_hgq_torch.py create mode 100644 tests/test_torch_pruning_layers.py diff --git a/src/pquant/__init__.py b/src/pquant/__init__.py index 299fa45..c5b9b04 100644 --- a/src/pquant/__init__.py +++ b/src/pquant/__init__.py @@ -5,7 +5,7 @@ # flake8: noqa backend = os.getenv("KERAS_BACKEND", "tensorflow") if backend == "torch": - from . import configs, pruning_methods + from . import configs from .core.hyperparameter_optimization import ( PQConfig, ap_config, @@ -19,7 +19,7 @@ pdp_config, wanda_config, ) - from .core.torch import activations, layers, optimizers, quantizer + from .core.torch import activations, layers, optimizers, pruning_methods, quantizer from .core.torch.layers import ( add_compression_layers, apply_final_compression, @@ -61,7 +61,7 @@ __all__ = _forwards else: - from . import configs, pruning_methods + from . import configs from .core.hyperparameter_optimization import ( PQConfig, ap_config, @@ -74,7 +74,7 @@ pdp_config, wanda_config, ) - from .core.keras import activations, layers, quantizer + from .core.keras import activations, layers, pruning_methods, quantizer from .core.keras.layers import ( add_compression_layers, apply_final_compression, diff --git a/src/pquant/core/constants.py b/src/pquant/core/constants.py index 993042b..6714d98 100644 --- a/src/pquant/core/constants.py +++ b/src/pquant/core/constants.py @@ -10,15 +10,6 @@ PDPPruningModel, WandaPruningModel, ) -from pquant.pruning_methods.constraint_functions import ( - EqualityConstraint, - GreaterThanOrEqualConstraint, - LessThanOrEqualConstraint, -) -from pquant.pruning_methods.metric_functions import ( - StructuredSparsityMetric, - UnstructuredSparsityMetric, -) PRUNING_MODEL_REGISTRY = { "cs": CSPruningModel, @@ -53,15 +44,3 @@ CONFIG_FILE = "config.yaml" N_JOBS = 1 - - -METRIC_REGISTRY = { - "UnstructuredSparsity": UnstructuredSparsityMetric, - "StructuredSparsity": StructuredSparsityMetric, -} - -CONSTRAINT_REGISTRY = { - "Equality": EqualityConstraint, - "LessThanOrEqual": LessThanOrEqualConstraint, - "GreaterThanOrEqual": GreaterThanOrEqualConstraint, -} diff --git a/src/pquant/core/keras/layers.py b/src/pquant/core/keras/layers.py index aabf69f..17d5051 100644 --- a/src/pquant/core/keras/layers.py +++ b/src/pquant/core/keras/layers.py @@ -25,7 +25,7 @@ from pquant.core.hyperparameter_optimization import PQConfig from pquant.core.keras.activations import PQActivation from pquant.core.keras.quantizer import Quantizer -from pquant.core.utils import get_pruning_layer +from pquant.core.keras.utils import get_pruning_layer T = TypeVar("T") diff --git a/src/pquant/pruning_methods/__init__.py b/src/pquant/core/keras/pruning_methods/__init__.py similarity index 100% rename from src/pquant/pruning_methods/__init__.py rename to src/pquant/core/keras/pruning_methods/__init__.py diff --git a/src/pquant/pruning_methods/activation_pruning.py b/src/pquant/core/keras/pruning_methods/activation_pruning.py similarity index 100% rename from src/pquant/pruning_methods/activation_pruning.py rename to src/pquant/core/keras/pruning_methods/activation_pruning.py diff --git a/src/pquant/pruning_methods/autosparse.py b/src/pquant/core/keras/pruning_methods/autosparse.py similarity index 97% rename from src/pquant/pruning_methods/autosparse.py rename to src/pquant/core/keras/pruning_methods/autosparse.py index 07c1252..910f09a 100644 --- a/src/pquant/pruning_methods/autosparse.py +++ b/src/pquant/core/keras/pruning_methods/autosparse.py @@ -113,7 +113,9 @@ def call(self, weight): is_training = ops.logical_not(ops.logical_or(self.is_pretraining, self.is_finetuning)) self.mask.assign(ops.where(is_training, new_binary_mask, ops.convert_to_tensor(self.mask))) - sparse_weight = ops.sign(weight) * ops.reshape(autosparse_prune(w_t, self.alpha), weight.shape) + sparse_weight = ops.sign(weight) * ops.reshape( + autosparse_prune(w_t, ops.convert_to_tensor(self.alpha)), weight.shape + ) return ops.where( self.is_pretraining, diff --git a/src/pquant/pruning_methods/constraint_functions.py b/src/pquant/core/keras/pruning_methods/constraint_functions.py similarity index 100% rename from src/pquant/pruning_methods/constraint_functions.py rename to src/pquant/core/keras/pruning_methods/constraint_functions.py diff --git a/src/pquant/pruning_methods/cs.py b/src/pquant/core/keras/pruning_methods/cs.py similarity index 100% rename from src/pquant/pruning_methods/cs.py rename to src/pquant/core/keras/pruning_methods/cs.py diff --git a/src/pquant/pruning_methods/dst.py b/src/pquant/core/keras/pruning_methods/dst.py similarity index 100% rename from src/pquant/pruning_methods/dst.py rename to src/pquant/core/keras/pruning_methods/dst.py diff --git a/src/pquant/pruning_methods/mdmm.py b/src/pquant/core/keras/pruning_methods/mdmm.py similarity index 77% rename from src/pquant/pruning_methods/mdmm.py rename to src/pquant/core/keras/pruning_methods/mdmm.py index 5837dd0..62b5e01 100644 --- a/src/pquant/pruning_methods/mdmm.py +++ b/src/pquant/core/keras/pruning_methods/mdmm.py @@ -8,7 +8,26 @@ import keras from keras import ops -from pquant.core.constants import CONSTRAINT_REGISTRY, METRIC_REGISTRY +from pquant.core.keras.pruning_methods.constraint_functions import ( + EqualityConstraint, + GreaterThanOrEqualConstraint, + LessThanOrEqualConstraint, +) +from pquant.core.keras.pruning_methods.metric_functions import ( + StructuredSparsityMetric, + UnstructuredSparsityMetric, +) + +METRIC_REGISTRY = { + "UnstructuredSparsity": UnstructuredSparsityMetric, + "StructuredSparsity": StructuredSparsityMetric, +} + +CONSTRAINT_REGISTRY = { + "Equality": EqualityConstraint, + "LessThanOrEqual": LessThanOrEqualConstraint, + "GreaterThanOrEqual": GreaterThanOrEqualConstraint, +} # ------------------------------------------------------------------- # MDMM Layer @@ -28,6 +47,10 @@ def __init__(self, config, layer_type, *args, **kwargs): self.constraint_layer = None self._is_finetuning = False self._is_pretraining = True + # TEMP: cache last penalty so calculate_additional_loss() works in + # custom training loops via get_model_losses(). Remove once the + # add_loss()/model.fit path is the only supported path. + self._last_penalty = None def build(self, input_shape): pruning_parameters = self.config.pruning_parameters @@ -94,8 +117,11 @@ def call(self, weight): self.mask.assign(ops.where(not_active, ops.convert_to_tensor(self.mask), hard_mask)) penalty = ops.sum(self.constraint_layer(weight)) - self.add_loss(ops.where(not_active, ops.zeros_like(penalty), penalty)) - + gated_penalty = ops.where(not_active, ops.zeros_like(penalty), penalty) + self.add_loss(gated_penalty) + # TEMP: cache for calculate_additional_loss() β€” remove with the + # _last_penalty attribute once custom-loop callers move to model.losses. + self._last_penalty = gated_penalty return ops.where(self.is_finetuning, weight * hard_mask, weight) def get_hard_mask(self, weight=None): @@ -109,7 +135,12 @@ def get_layer_sparsity(self, weight): def calculate_additional_loss(self): # Loss is added via self.add_loss() in call() for model.fit. - # For custom training loops, accumulate model.losses from the last forward pass instead. + # TEMP: also return the cached penalty so custom training loops using + # get_model_losses() see the constraint term. Remove this branch (and + # the _last_penalty cache) once those callers switch to model.losses; + # then this can revert to `return 0.0`. + if self._last_penalty is not None: + return self._last_penalty return 0.0 def pre_epoch_function(self, epoch, total_epochs): diff --git a/src/pquant/pruning_methods/metric_functions.py b/src/pquant/core/keras/pruning_methods/metric_functions.py similarity index 100% rename from src/pquant/pruning_methods/metric_functions.py rename to src/pquant/core/keras/pruning_methods/metric_functions.py diff --git a/src/pquant/pruning_methods/pdp.py b/src/pquant/core/keras/pruning_methods/pdp.py similarity index 100% rename from src/pquant/pruning_methods/pdp.py rename to src/pquant/core/keras/pruning_methods/pdp.py diff --git a/src/pquant/pruning_methods/wanda.py b/src/pquant/core/keras/pruning_methods/wanda.py similarity index 100% rename from src/pquant/pruning_methods/wanda.py rename to src/pquant/core/keras/pruning_methods/wanda.py diff --git a/src/pquant/core/keras/quantizer.py b/src/pquant/core/keras/quantizer.py index 38b96af..8cc63af 100644 --- a/src/pquant/core/keras/quantizer.py +++ b/src/pquant/core/keras/quantizer.py @@ -1,9 +1,10 @@ from enum import Enum import keras +from hgq.quantizer import Quantizer as HGQQuantizer +from hgq.quantizer import QuantizerConfig from keras import ops - -from pquant.core.quantizer_functions import create_quantizer +from quantizers import get_fixed_quantizer @keras.saving.register_keras_serializable(package="PQuantML") @@ -184,3 +185,34 @@ def get_config(self): if self.use_hgq: config.update({"quantizer": keras.saving.serialize_keras_object(self.quantizer)}) return config + + +def create_hgq_parameters_quantizer(k, i, f, overflow, round_mode, place, gamma=1e-8): + quantizer_config = QuantizerConfig( + q_type="kif", place=place, k0=k, i0=i, f0=f, overflow_mode=overflow, round_mode=round_mode, homogeneous_axis=() + ) + return HGQQuantizer(config=quantizer_config) + + +def create_hgq_data_quantizer(k, i, f, overflow, round_mode, gamma=1e-8): + quantizer_config = QuantizerConfig( + q_type="kif", + place="datalane", + k0=k, + i0=i, + f0=f, + overflow_mode=overflow, + round_mode=round_mode, + homogeneous_axis=(0,), + ) + return HGQQuantizer(config=quantizer_config) + + +def create_quantizer(k, i, f, overflow, round_mode, is_heterogeneous, is_data, place="datalane", gamma=1e-8): + if is_heterogeneous: + if is_data: + return create_hgq_data_quantizer(k, i, f, overflow, round_mode, gamma=gamma) + else: + return create_hgq_parameters_quantizer(k, i, f, overflow, round_mode, place, gamma=gamma) + else: + return get_fixed_quantizer(round_mode=round_mode, overflow_mode=overflow) diff --git a/src/pquant/core/keras/utils.py b/src/pquant/core/keras/utils.py new file mode 100644 index 0000000..a0e4faa --- /dev/null +++ b/src/pquant/core/keras/utils.py @@ -0,0 +1,25 @@ +from pquant.core.keras.pruning_methods.activation_pruning import ActivationPruning +from pquant.core.keras.pruning_methods.autosparse import AutoSparse +from pquant.core.keras.pruning_methods.cs import ContinuousSparsification +from pquant.core.keras.pruning_methods.dst import DST +from pquant.core.keras.pruning_methods.mdmm import MDMM +from pquant.core.keras.pruning_methods.pdp import PDP +from pquant.core.keras.pruning_methods.wanda import Wanda + + +def get_pruning_layer(config, layer_type): + pruning_method = config.pruning_parameters.pruning_method + if pruning_method == "dst": + return DST(config, layer_type) + elif pruning_method == "autosparse": + return AutoSparse(config, layer_type) + elif pruning_method == "cs": + return ContinuousSparsification(config, layer_type) + elif pruning_method == "pdp": + return PDP(config, layer_type) + elif pruning_method == "activation_pruning": + return ActivationPruning(config, layer_type) + elif pruning_method == "wanda": + return Wanda(config, layer_type) + elif pruning_method == "mdmm": + return MDMM(config, layer_type) diff --git a/src/pquant/core/quantizer_functions.py b/src/pquant/core/quantizer_functions.py deleted file mode 100644 index 81922d7..0000000 --- a/src/pquant/core/quantizer_functions.py +++ /dev/null @@ -1,48 +0,0 @@ -import keras - - -def create_fixed_quantizer(k, i, f, overflow, round_mode): - if keras.backend.backend() == "torch": - from pquant.core.torch.fixed_point_quantizer import get_fixed_quantizer - else: - from quantizers import get_fixed_quantizer - - quantizer = get_fixed_quantizer(round_mode=round_mode, overflow_mode=overflow) - return quantizer - - -def create_hgq_parameters_quantizer(k, i, f, overflow, round_mode, place): - from hgq.quantizer import Quantizer, QuantizerConfig - - quantizer_config = QuantizerConfig( - q_type="kif", place=place, k0=k, i0=i, f0=f, overflow_mode=overflow, round_mode=round_mode, homogeneous_axis=() - ) - - return Quantizer(config=quantizer_config) - - -def create_hgq_data_quantizer(k, i, f, overflow, round_mode): - from hgq.quantizer import Quantizer, QuantizerConfig - - quantizer_config = QuantizerConfig( - q_type="kif", - place="datalane", - k0=k, - i0=i, - f0=f, - overflow_mode=overflow, - round_mode=round_mode, - homogeneous_axis=(0,), - ) - - return Quantizer(config=quantizer_config) - - -def create_quantizer(k, i, f, overflow, round_mode, is_heterogeneous, is_data, place="datalane"): - if is_heterogeneous: - if is_data: - return create_hgq_data_quantizer(k, i, f, overflow, round_mode) - else: - return create_hgq_parameters_quantizer(k, i, f, overflow, round_mode, place) - else: - return create_fixed_quantizer(k, i, f, overflow, round_mode) diff --git a/src/pquant/core/torch/hgq_quantizer.py b/src/pquant/core/torch/hgq_quantizer.py new file mode 100644 index 0000000..c10ab55 --- /dev/null +++ b/src/pquant/core/torch/hgq_quantizer.py @@ -0,0 +1,408 @@ +""" +Pure-PyTorch implementation of HGQ FixedPointQuantizerKIF + Quantizer wrapper. + +Replaces the Keras hgq2 library dependency for the PyTorch backend. +Combines DefaultBitwidthMapper + FixedPointQuantizerKIF + Quantizer into one +nn.Module with no inheritance chain. + +Reference: "HGQ: High Granularity Quantization for Real-time Neural Networks on FPGAs" + (Sun et al., FPGA '26) +""" + +import logging +import math + +import torch +import torch.nn as nn + +from pquant.core.torch.fixed_point_quantizer import get_fixed_quantizer, round_conv + +logger = logging.getLogger(__name__) + + +def _minimal_i_given_xf(absmax: torch.Tensor, f: torch.Tensor) -> torch.Tensor: + """Minimum integer bits needed to represent absmax with f fractional bits.""" + eps = 2.0 ** (-f) + return torch.ceil(torch.log2(absmax + eps + 1e-10)) + + +class HGQQuantizer(nn.Module): + """ + HGQ fixed-point quantizer parameterized by (k, i, f) β€” keep_negative, integer + bits, fractional bits. + + Parameters + ---------- + k0 : float + Initial sign bit (0 = unsigned, 1 = signed). Non-trainable. + i0 : float + Initial integer bits. Trainable for SAT; tracked buffer for WRAP. + f0 : float + Initial fractional bits. Always trainable. + overflow_mode : str + One of 'SAT', 'SAT_SYM', 'WRAP', 'WRAP_SM'. + round_mode : str + One of 'RND', 'RND_CONV', 'TRN', etc. + is_data : bool + True β†’ data/activation quantizer (homogeneous over batch axis 0). + False β†’ weight/bias quantizer (fully heterogeneous, per-element). + gamma : float + L1 regularisation coefficient on bit-widths. + i_decay_speed : float + WRAP mode only. Rate at which tracked i can decrease per step. + float('inf') means i is reset each step to the minimum required by data. + i_min, i_max : float + Clamp bounds for the i parameter / buffer. + f_min, f_max : float + Clamp bounds for the f parameter. + scaler : float | None + Optional: inputs are divided by scaler before quantisation and multiplied + after (equivalent to scaling the fixed-point range). + qnoise_factor : float | None + Optional: mix factor for quantisation noise injection during training. + output = input + qnoise_factor * (quantised - input). + affine : tuple[float, float] | None + Optional (scale, shift) applied after quantisation: out = out*scale + shift. + """ + + def __init__( + self, + k0: float, + i0: float, + f0: float, + overflow_mode: str, + round_mode: str, + is_data: bool, + gamma: float = 1e-8, + i_decay_speed: float = float("inf"), + i_min: float = -23.0, + i_max: float = 23.0, + f_min: float = -24.0, + f_max: float = 24.0, + scaler=None, + qnoise_factor: float | None = None, + affine=None, + ): + super().__init__() + assert int(k0) in (0, 1), f"k0 must be 0 or 1, got {k0}" + + self.k0 = float(k0) + self.i0 = float(i0) + self.f0 = float(f0) + self.overflow_mode = overflow_mode.upper() + self.round_mode = round_mode.upper() + self.is_data = is_data + self.gamma = gamma + self.i_decay_speed = i_decay_speed + self.i_min = i_min + self.i_max = i_max + self.f_min = f_min + self.f_max = f_max + self.scaler = scaler + self.qnoise_factor = qnoise_factor + self.affine = affine + + # Set during build() + self.homogeneous_axis: tuple[int, ...] = () + self._built = False + + self._stateless_quantizer = get_fixed_quantizer(round_mode=round_mode, overflow_mode=overflow_mode) + + # Scalar placeholders β€” replaced with shaped tensors in build(). + # k and i_raw are non-trainable buffers; f (and i for SAT) are Parameters. + self.register_buffer("_k", torch.tensor(self.k0)) + self._f = nn.Parameter(torch.tensor(self.f0)) + + if self.overflow_mode == "WRAP": + # Integer bits tracked as non-trainable running buffer (not optimised). + self.register_buffer("_i_raw", torch.tensor(self.i0)) + else: + # Integer bits are trainable for SAT / SAT_SYM. + self._i = nn.Parameter(torch.tensor(self.i0)) + + def build(self, input_shape: tuple) -> None: + """ + Initialise shaped parameter / buffer tensors. + + Called automatically on the first forward pass. The optimizer must be + created *after* build() has been called so that it tracks the shaped + parameters, not the scalar placeholders from __init__. + """ + device = self._k.device + bw_shape = self._infer_bw_shape(input_shape) + + self.homogeneous_axis = (0,) if self.is_data else () + + # k: non-trainable sign-bit buffer + self.register_buffer("_k", torch.full(bw_shape, self.k0, device=device)) + + # f: always trainable + self._f = nn.Parameter(torch.full(bw_shape, self.f0, device=device)) + + if self.overflow_mode == "WRAP": + self.register_buffer("_i_raw", torch.full(bw_shape, self.i0, device=device)) + else: + self._i = nn.Parameter(torch.full(bw_shape, self.i0, device=device)) + + self._built = True + + def _infer_bw_shape(self, input_shape: tuple) -> tuple: + """Shape of bit-width parameter tensors given input tensor shape.""" + if self.is_data: + # Batch axis (0) is homogeneous β†’ dimension 0 collapses to 1. + shape = list(input_shape) + shape[0] = 1 + return tuple(shape) + # Fully heterogeneous (per-parameter): same shape as input. + return tuple(input_shape) + + # ------------------------------------------------------------------ + # Properties + # ------------------------------------------------------------------ + + @property + def built(self) -> bool: + return self._built + + @property + def k(self) -> torch.Tensor: + return self._k.float() + + @property + def i(self) -> torch.Tensor: + """Integer bits, rounded with STE.""" + if self.overflow_mode == "WRAP": + # _i_raw is a buffer (no gradient); round_conv is still used for + # consistency β€” it degenerates to plain rounding when grad=0. + return round_conv(self._i_raw.float()) + return round_conv(self._i.float()) + + @property + def f(self) -> torch.Tensor: + """Fractional bits, rounded with STE.""" + return round_conv(self._f.float()) + + @property + def b(self) -> torch.Tensor: + """Total non-sign bits = relu(i + f). Zero when quantizer is pruned.""" + return torch.relu(self.i + self.f) + + # ------------------------------------------------------------------ + # Bit-width mapping helpers + # ------------------------------------------------------------------ + + def _bw_to_x(self, bw: torch.Tensor, x_shape: tuple) -> torch.Tensor: + return bw.expand(x_shape) + + def _x_to_bw_absmax(self, x: torch.Tensor) -> torch.Tensor: + if len(self.homogeneous_axis) == 0: + return x.abs() + return torch.amax(x.abs(), dim=self.homogeneous_axis, keepdim=True) + + # ------------------------------------------------------------------ + # Forward pass + # ------------------------------------------------------------------ + + def forward(self, x: torch.Tensor, training: bool = False) -> torch.Tensor: + if not self._built: + self.build(tuple(x.shape)) + + if training: + with torch.no_grad(): + self._f.data.clamp_(self.f_min, self.f_max) + if self.overflow_mode != "WRAP": + self._i.data.clamp_(self.i_min, self.i_max) + + x_in = x # kept for qnoise + + if self.scaler is not None: + x = x / self.scaler + + # Round bit-width parameters to integers with STE so gradients flow. + f_bw = round_conv(self._f) # bw-shaped + k = self._k.float() + f_x = f_bw.expand(x.shape) + k_x = k.expand(x.shape) + + if self.overflow_mode == "WRAP": + out = self._stateless_quantizer.round(x, f_x) + + if training: + # Track minimum integer bits needed for the current data. + # Done without gradient so it doesn't affect the f gradient path. + with torch.no_grad(): + absmax = self._x_to_bw_absmax(out.detach()) + min_i = _minimal_i_given_xf(absmax, f_bw.detach()) + if math.isinf(self.i_decay_speed): + new_i = min_i + else: + new_i = torch.maximum(self._i_raw - self.i_decay_speed, min_i) + self._i_raw.copy_(new_i.clamp(self.i_min, self.i_max)) + else: + if self.is_data: + # Data quantizer: apply wrap-modulo after rounding (inference). + i_x = self.i.expand(x.shape) + out = self._stateless_quantizer.saturate(out, k_x, i_x, f_x) + # Weight quantizer: rounded output is final, no saturation. + + # Zero out pruned (total bits == 0) values. + i_x = self.i.expand(x.shape) + out = torch.where(k_x + i_x + f_x > 0, out, torch.zeros_like(out)) + + else: # SAT / SAT_SYM + i_bw = round_conv(self._i) # bw-shaped, STE for gradient + i_x = i_bw.expand(x.shape) + out = self._stateless_quantizer(x, k_x, i_x, f_x, training) + + if not training: + out = torch.where(k_x + i_x + f_x > 0, out, torch.zeros_like(out)) + + if self.scaler is not None: + out = out * self.scaler + + if self.qnoise_factor is not None and training: + out = x_in + self.qnoise_factor * (out - x_in) + + if self.affine is not None: + out = out * self.affine[0] + self.affine[1] + + return out + + # ------------------------------------------------------------------ + # Regularisation / constraint / utility methods + # ------------------------------------------------------------------ + + def regularization_loss(self) -> torch.Tensor: + """ + L1 regularisation on bit-widths. + + Replaces Keras `layer.losses` list. Contributes the Ξ³Β·Ξ£(bit-widths) term + from eq. (12) in the paper. For SAT mode both i and f are regularised; + for WRAP mode only f (i is not a learnable parameter). + """ + if not self._built or self.gamma == 0.0: + return torch.tensor(0.0, device=self._k.device) + loss = self.gamma * self.f.sum() + if self.overflow_mode != "WRAP": + loss = loss + self.gamma * self.i.sum() + return loss + + def post_epoch_constraint_apply(self) -> None: + """ + Clamp i and f parameters to [*_min, *_max] in-place. + + Replaces Keras `variable.constraint(variable)` + `variable.assign()`. + Call at the end of each epoch. + """ + with torch.no_grad(): + self._f.data.clamp_(self.f_min, self.f_max) + if self.overflow_mode != "WRAP": + self._i.data.clamp_(self.i_min, self.i_max) + # WRAP: _i_raw is already clamped inside forward(). + + def set_bits(self, i, f) -> None: + """ + Overwrite i and f with scalar or tensor values. + + Replaces Keras `variable.assign(variable * 0 + value)` pattern. + Used by `apply_final_compression` and `reload_from_local` in quantizer.py. + """ + with torch.no_grad(): + i_t = torch.as_tensor(i, dtype=torch.float32) + f_t = torch.as_tensor(f, dtype=torch.float32) + self._f.data.copy_(f_t) if f_t.shape == self._f.shape else self._f.data.fill_(f_t.item()) + if self.overflow_mode == "WRAP": + self._i_raw.copy_(i_t) if i_t.shape == self._i_raw.shape else self._i_raw.fill_(i_t.item()) + else: + self._i.data.copy_(i_t) if i_t.shape == self._i.shape else self._i.data.fill_(i_t.item()) + + def bits_(self, shape: tuple) -> torch.Tensor: + """ + Return total bits (k + relu(i+f)) broadcast to *shape*. + + Used by `Quantizer.get_total_bits()` for EBOPs calculations. + """ + if not self._built: + total = self.k0 + max(self.i0 + self.f0, 0.0) + return torch.full(shape, total, device=self._k.device) + return (self.k + self.b).expand(shape) + + +# --------------------------------------------------------------------------- +# Quick sanity check +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + torch.manual_seed(0) + logging.basicConfig(level=logging.DEBUG, format="%(message)s") + logger.debug("=== HGQQuantizer sanity checks ===\n") + + # ---- SAT, signed, weight quantizer (per-element) ---- + x = torch.randn(4, 3) * 0.5 + q_sat = HGQQuantizer(k0=1, i0=2, f0=4, overflow_mode="SAT", round_mode="RND", is_data=False) + out = q_sat(x, training=False) + logger.debug(f"SAT weight forward: input range [{x.min():.3f}, {x.max():.3f}]") + logger.debug(f" output range [{out.min():.3f}, {out.max():.3f}]") + max_val = 2**2 - 2**-4 + assert out.max() <= max_val + 1e-5, "SAT upper bound violated" + assert out.min() >= -(2**2) - 1e-5, "SAT lower bound violated" + logger.debug(" bounds check OK\n") + + # ---- SAT backward (gradient through f) ---- + q_sat2 = HGQQuantizer(k0=1, i0=2, f0=4, overflow_mode="SAT", round_mode="RND", is_data=False) + x2 = torch.randn(4, 3, requires_grad=True) + out2 = q_sat2(x2, training=True) + out2.sum().backward() + assert q_sat2._f.grad is not None and q_sat2._f.grad.abs().sum() > 0 + assert x2.grad is not None and x2.grad.abs().sum() > 0 + logger.debug("SAT backward: gradients w.r.t. f and input OK") + logger.debug(f" f grad mean: {q_sat2._f.grad.mean():.6f}\n") + + # ---- WRAP, signed, data quantizer (per-batch) ---- + x3 = torch.randn(8, 4) * 3.0 + q_wrap = HGQQuantizer(k0=1, i0=0, f0=4, overflow_mode="WRAP", round_mode="RND", is_data=True) + out3 = q_wrap(x3, training=True) + logger.debug(f"WRAP data training: _i_raw after forward: {q_wrap._i_raw.flatten()[:4]}") + assert q_wrap._i_raw.max() >= 0, "i_raw should track positive range" + logger.debug(" _i_raw tracking OK\n") + + # ---- WRAP backward (gradient through f, not i_raw) ---- + q_wrap2 = HGQQuantizer(k0=1, i0=2, f0=4, overflow_mode="WRAP", round_mode="RND", is_data=False) + x4 = torch.randn(4, 3, requires_grad=True) + out4 = q_wrap2(x4, training=True) + out4.sum().backward() + assert q_wrap2._f.grad is not None and q_wrap2._f.grad.abs().sum() > 0 + assert q_wrap2._i_raw.grad is None, "_i_raw must not accumulate gradient" + logger.debug("WRAP backward: f has gradient, _i_raw has none OK\n") + + # ---- Zero bits β†’ pruned: k=0,i=0,f=0 β†’ k+i+f=0 β†’ all zero ---- + q_pruned = HGQQuantizer(k0=0, i0=0, f0=0, overflow_mode="SAT", round_mode="RND", is_data=False) + x5 = torch.randn(4, 3) + out5 = q_pruned(x5, training=False) + assert torch.all(out5 == 0), "Zero-bit quantizer should output zeros" + logger.debug("Pruning (zero bits): output is all zeros OK\n") + + # ---- regularization_loss ---- + q_reg = HGQQuantizer(k0=1, i0=2.0, f0=4.0, overflow_mode="SAT", round_mode="RND", is_data=False, gamma=1e-3) + q_reg(torch.randn(4, 3)) + loss = q_reg.regularization_loss() + expected = 1e-3 * (q_reg.f.sum() + q_reg.i.sum()) + assert torch.isclose(loss, expected, rtol=1e-4), f"reg loss mismatch: {loss} vs {expected}" + logger.debug(f"regularization_loss OK: {loss.item():.6f}\n") + + # ---- bits_ shape ---- + q_bits = HGQQuantizer(k0=1, i0=2.0, f0=4.0, overflow_mode="SAT", round_mode="RND", is_data=False) + q_bits(torch.randn(4, 3)) + b = q_bits.bits_((8, 4, 3)) + assert b.shape == (8, 4, 3), f"bits_ shape wrong: {b.shape}" + logger.debug(f"bits_ shape OK: {b.shape}\n") + + # ---- set_bits ---- + q_set = HGQQuantizer(k0=1, i0=2.0, f0=4.0, overflow_mode="SAT", round_mode="RND", is_data=False) + q_set(torch.randn(4, 3)) + q_set.set_bits(3.0, 5.0) + assert math.isclose(q_set.i.mean().item(), 3.0, abs_tol=1e-5) + assert math.isclose(q_set.f.mean().item(), 5.0, abs_tol=1e-5) + logger.debug("set_bits OK\n") + + logger.debug("=== All checks passed ===") diff --git a/src/pquant/core/torch/layers.py b/src/pquant/core/torch/layers.py index 87ced7c..83235d1 100644 --- a/src/pquant/core/torch/layers.py +++ b/src/pquant/core/torch/layers.py @@ -9,7 +9,7 @@ from pquant.core.torch.activations import PQActivation from pquant.core.torch.quantizer import Quantizer -from pquant.core.utils import get_pruning_layer +from pquant.core.torch.utils import get_pruning_layer if typing.TYPE_CHECKING: from pquant.core.torch.fit_compress import call_fitcompress # noqa: 401 @@ -170,6 +170,8 @@ def save_weights(self): self.init_weight = self._weight.clone() def rewind_weights(self): + if not hasattr(self, "init_weight"): + return self._weight.data = self.init_weight.clone() def ebops(self): @@ -1513,7 +1515,8 @@ def post_pretrain_functions(model, config, train_loader=None, loss_function=None if isinstance(layer, (PQConv2d, PQConv1d, PQDense)): # layer.post_pre_train_function() # set_data_quantization_bits(model) - layer.pruning_layer.mask.assign(pruning_mask_importance_scores[idx]) + with torch.no_grad(): + layer.pruning_layer.mask.data = pruning_mask_importance_scores[idx] layer.pruning_layer.pre_finetune_function() # So mask is not updated during training anymore idx += 1 return diff --git a/src/pquant/core/torch/pruning_methods/__init__.py b/src/pquant/core/torch/pruning_methods/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pquant/core/torch/pruning_methods/activation_pruning.py b/src/pquant/core/torch/pruning_methods/activation_pruning.py new file mode 100644 index 0000000..554a4b5 --- /dev/null +++ b/src/pquant/core/torch/pruning_methods/activation_pruning.py @@ -0,0 +1,104 @@ +import torch +import torch.nn as nn + + +class ActivationPruning(nn.Module): + def __init__(self, config, layer_type, *args, **kwargs): + super().__init__() + if isinstance(config, dict): + from pquant.core.hyperparameter_optimization import PQConfig + + config = PQConfig.load_from_config(config) + self.config = config + self.act_type = "relu" + self.layer_type = layer_type + self._is_pretraining = True + self._is_finetuning = False + self.is_pretraining = True + self.is_finetuning = False + self.threshold = float(config.pruning_parameters.threshold) + self.t_start_collecting_batch = int(self.config.pruning_parameters.t_start_collecting_batch) + self.built = False + + def build(self, input_shape): + if self.built: + return + if self.layer_type in ("conv", "depthwise_conv"): + if len(input_shape) == 3: + shape = (input_shape[0], 1, 1) + else: + shape = (input_shape[0], 1, 1, 1) + else: + shape = (input_shape[0], 1) + self.shape = shape + n_channels = input_shape[0] + self.register_buffer("mask", torch.ones(shape)) + self.register_buffer("mask_placeholder", torch.ones(shape)) + self.register_buffer("activations", torch.zeros(n_channels)) + self.register_buffer("batches_collected", torch.zeros((), dtype=torch.int32)) + self.register_buffer("t", torch.zeros((), dtype=torch.int32)) + self.built = True + + @torch.no_grad() + def collect_output(self, output, training): + if not training: + return + if self._is_pretraining or self._is_finetuning: + return + if int(self.t.item()) < self.t_start_collecting_batch: + return + + t_delta = int(self.config.pruning_parameters.t_delta) + + gt_zero = (output > 0).to(output.dtype) + if self.layer_type == "linear": + per_channel = gt_zero.mean(dim=0) + else: + axes = (0,) + tuple(range(2, output.dim())) + per_channel = gt_zero.mean(dim=axes) + + self.activations.add_(per_channel.to(self.activations.dtype)) + self.batches_collected.add_(1) + + if int(self.batches_collected.item()) % t_delta == 0: + denom = max(int(self.batches_collected.item()), 1) + pct_active = self.activations / denom + new_mask = (pct_active > self.threshold).to(self.mask_placeholder.dtype).reshape(self.shape) + self.mask_placeholder.copy_(new_mask) + self.activations.zero_() + self.batches_collected.zero_() + self.t.zero_() + + def forward(self, weight): + if self._is_pretraining: + return weight + return self.mask.to(weight.dtype) * weight + + def get_hard_mask(self, weight=None): + return self.mask + + def post_pre_train_function(self): + self._is_pretraining = False + self.is_pretraining = False + + def pre_epoch_function(self, epoch, total_epochs, **kwargs): + pass + + def post_round_function(self): + pass + + def pre_finetune_function(self): + self._is_finetuning = True + self.is_finetuning = True + + def calculate_additional_loss(self): + return 0.0 + + def get_layer_sparsity(self, weight): + pass + + @torch.no_grad() + def post_epoch_function(self, epoch, total_epochs, **kwargs): + if not self._is_pretraining: + self.t.add_(1) + self.mask.copy_(self.mask_placeholder) diff --git a/src/pquant/core/torch/pruning_methods/autosparse.py b/src/pquant/core/torch/pruning_methods/autosparse.py new file mode 100644 index 0000000..81b723b --- /dev/null +++ b/src/pquant/core/torch/pruning_methods/autosparse.py @@ -0,0 +1,143 @@ +import math + +import numpy as np +import torch +import torch.nn as nn + +_PI = math.pi +_L0 = -6.0 +_L1 = 6.0 + + +def cosine_decay(i, T): + return (1 + math.cos(_PI * i / T)) / 2 + + +def sigmoid_decay(i, T): + x = _L0 + (_L1 - _L0) * i / T + return 1.0 - 1.0 / (1.0 + math.exp(-x)) + + +def cosine_sigmoid_decay(i, T): + return max(cosine_decay(i, T), sigmoid_decay(i, T)) + + +def get_threshold_size(config, weight_shape): + if config.pruning_parameters.threshold_type == "layerwise": + return (1, 1) + elif config.pruning_parameters.threshold_type == "channelwise": + return (weight_shape[0], 1) + elif config.pruning_parameters.threshold_type == "weightwise": + return (weight_shape[0], int(np.prod(weight_shape[1:]))) + + +class _AutoSparsePrune(torch.autograd.Function): + @staticmethod + def forward(ctx, x, alpha, backward_sparsity_flag, backward_sparsity): + mask = torch.relu(x) + kth_value = None + if backward_sparsity_flag: + flat = x.reshape(-1) + k = max(int(flat.numel() * backward_sparsity), 1) + topk_vals, _ = torch.topk(flat, k) + kth_value = topk_vals[-1] + ctx.save_for_backward(x, alpha, kth_value if kth_value is not None else torch.zeros((), device=x.device)) + ctx.backward_sparsity_flag = backward_sparsity_flag + return mask + + @staticmethod + def backward(ctx, upstream): + x, alpha, kth_value = ctx.saved_tensors + grads = torch.where(x <= 0, alpha.to(x.dtype), torch.ones_like(x)) + if ctx.backward_sparsity_flag: + grads = torch.where(x < kth_value, torch.zeros_like(grads), grads) + return grads * upstream, None, None, None + + +def autosparse_prune(x, alpha, backward_sparsity_flag, backward_sparsity): + return _AutoSparsePrune.apply(x, alpha, backward_sparsity_flag, backward_sparsity) + + +class AutoSparse(nn.Module): + def __init__(self, config, layer_type, *args, **kwargs): + super().__init__() + if isinstance(config, dict): + from pquant.core.hyperparameter_optimization import PQConfig + + config = PQConfig.load_from_config(config) + self.config = config + self.layer_type = layer_type + self._alpha_init = float(config.pruning_parameters.alpha) + self._backward_sparsity_flag = bool(config.pruning_parameters.backward_sparsity) + self._backward_sparsity = 0.5 + self._is_pretraining = True + self._is_finetuning = False + self.is_pretraining = True + self.is_finetuning = False + self.built = False + + def build(self, input_shape): + if self.built: + return + threshold_size = get_threshold_size(self.config, input_shape) + self.threshold = nn.Parameter(torch.full(threshold_size, float(self.config.pruning_parameters.threshold_init))) + self.register_buffer("mask", torch.ones(tuple(input_shape))) + self.register_buffer("alpha", torch.tensor(self._alpha_init)) + self.built = True + + def _g(self, x): + return torch.sigmoid(x) + + def forward(self, weight): + if self._is_pretraining: + return weight + if self._is_finetuning: + return self.mask.to(weight.dtype) * weight + + weight_reshaped = weight.reshape(weight.shape[0], -1) + w_t = weight_reshaped.abs() - torch.sigmoid(self.threshold) + + new_binary_mask = (w_t > 0).to(weight.dtype).reshape(weight.shape) + with torch.no_grad(): + self.mask.copy_(new_binary_mask) + + sparse = torch.sign(weight) * autosparse_prune( + w_t, self.alpha, self._backward_sparsity_flag, self._backward_sparsity + ).reshape(weight.shape) + return sparse + + def get_hard_mask(self, weight=None): + return self.mask + + def get_mask(self, weight): + weight_reshaped = weight.reshape(weight.shape[0], -1) + w_t = weight_reshaped.abs() - torch.sigmoid(self.threshold) + return (w_t > 0).to(weight.dtype).reshape(weight.shape) + + def get_layer_sparsity(self, weight): + m = self.get_mask(weight) + return m.count_nonzero() / m.numel() + + def pre_epoch_function(self, epoch, total_epochs): + pass + + def calculate_additional_loss(self): + return 0.0 + + def pre_finetune_function(self): + self._is_finetuning = True + self.is_finetuning = True + + def post_round_function(self): + pass + + def post_pre_train_function(self): + self._is_pretraining = False + self.is_pretraining = False + + @torch.no_grad() + def post_epoch_function(self, epoch, total_epochs): + decay = cosine_sigmoid_decay(epoch, total_epochs) + self.alpha.fill_(self._alpha_init * decay) + if epoch >= self.config.pruning_parameters.alpha_reset_epoch: + self.alpha.zero_() diff --git a/src/pquant/core/torch/pruning_methods/constraint_functions.py b/src/pquant/core/torch/pruning_methods/constraint_functions.py new file mode 100644 index 0000000..565c960 --- /dev/null +++ b/src/pquant/core/torch/pruning_methods/constraint_functions.py @@ -0,0 +1,99 @@ +import abc + +import torch +import torch.nn as nn + + +class _FlipGradient(torch.autograd.Function): + @staticmethod + def forward(ctx, x, scale): + ctx.scale = float(scale) + return x + + @staticmethod + def backward(ctx, upstream): + return upstream * ctx.scale, None + + +def flip_gradient(x, scale=-1.0): + return _FlipGradient.apply(x, scale) + + +class Constraint(nn.Module): + def __init__(self, lmbda_init=1.0, scale=1.0, damping=1.0, use_grad=True, lr=0.0, **kwargs): + super().__init__() + self.use_grad_ = bool(use_grad) + self.lr_ = float(lr) + self.register_buffer("scale", torch.tensor(float(scale))) + self.register_buffer("damping", torch.tensor(float(damping))) + if self.use_grad_: + self.lmbda = nn.Parameter(torch.tensor(float(lmbda_init))) + else: + self.register_buffer("lmbda", torch.tensor(float(lmbda_init))) + self.register_buffer("prev_infs", torch.tensor(0.0)) + + def build(self, input_shape): + # Exists for API parity with the keras version β€” no-op in torch. + pass + + def forward(self, weight, training=None): + raw_infeasibility = self.get_infeasibility(weight) + infeasibility = self.pipe_infeasibility(raw_infeasibility) + + if self.use_grad_: + ascent_lmbda = flip_gradient(self.lmbda) + else: + lmbda_step = self.lr_ * self.scale * self.prev_infs + ascent_lmbda = self.lmbda + lmbda_step + if training: + with torch.no_grad(): + self.lmbda.add_(lmbda_step) + self.prev_infs.copy_(infeasibility.detach()) + + l_term = ascent_lmbda * infeasibility + damp_term = self.damping * infeasibility.square() / 2 + return self.scale * (l_term + damp_term) + + @abc.abstractmethod + def get_infeasibility(self, weight): + raise NotImplementedError + + def pipe_infeasibility(self, infeasibility): + return infeasibility + + @torch.no_grad() + def turn_off(self): + if not self.use_grad_: + self.lr_ = 0.0 + self.scale.zero_() + self.lmbda.data.zero_() if isinstance(self.lmbda, nn.Parameter) else self.lmbda.zero_() + + +class EqualityConstraint(Constraint): + def __init__(self, metric_fn, target_value=0.0, **kwargs): + super().__init__(**kwargs) + self.metric_fn = metric_fn + self.target_value = float(target_value) + + def get_infeasibility(self, weight): + return (self.metric_fn(weight) - self.target_value).abs() + + +class LessThanOrEqualConstraint(Constraint): + def __init__(self, metric_fn, target_value=0.0, **kwargs): + super().__init__(**kwargs) + self.metric_fn = metric_fn + self.target_value = float(target_value) + + def get_infeasibility(self, weight): + return torch.clamp(self.metric_fn(weight) - self.target_value, min=0.0) + + +class GreaterThanOrEqualConstraint(Constraint): + def __init__(self, metric_fn, target_value=0.0, **kwargs): + super().__init__(**kwargs) + self.metric_fn = metric_fn + self.target_value = float(target_value) + + def get_infeasibility(self, weight): + return torch.clamp(self.target_value - self.metric_fn(weight), min=0.0) diff --git a/src/pquant/core/torch/pruning_methods/cs.py b/src/pquant/core/torch/pruning_methods/cs.py new file mode 100644 index 0000000..3881ebc --- /dev/null +++ b/src/pquant/core/torch/pruning_methods/cs.py @@ -0,0 +1,79 @@ +import torch +import torch.nn as nn + + +class ContinuousSparsification(nn.Module): + def __init__(self, config, layer_type, *args, **kwargs): + super().__init__() + if isinstance(config, dict): + from pquant.core.hyperparameter_optimization import PQConfig + + config = PQConfig.load_from_config(config) + self.config = config + self.final_temp = float(config.pruning_parameters.final_temp) + self._is_finetuning = False + self._is_pretraining = True + self.is_pretraining = True + self.is_finetuning = False + self.layer_type = layer_type + self.built = False + + def build(self, input_shape): + if self.built: + return + init_val = float(self.config.pruning_parameters.threshold_init) + s_init = torch.full(tuple(input_shape), init_val) + self.s = nn.Parameter(s_init.clone()) + self.register_buffer("s_init", s_init.clone()) + self.register_buffer("scaling", 1.0 / torch.sigmoid(s_init)) + self.register_buffer("beta", torch.tensor(1.0)) + self.register_buffer("mask", torch.ones(tuple(input_shape))) + self.built = True + + def forward(self, weight): + if self._is_pretraining or self._is_finetuning: + return self.mask.to(weight.dtype) * weight + new_mask = self.get_mask() + with torch.no_grad(): + self.mask.copy_(new_mask.detach()) + return new_mask * weight + + def pre_finetune_function(self): + self._is_finetuning = True + self.is_finetuning = True + with torch.no_grad(): + self.mask.copy_(self.get_hard_mask().to(self.mask.dtype)) + + def get_mask(self): + return torch.sigmoid(self.beta * self.s) * self.scaling + + def post_pre_train_function(self): + self._is_pretraining = False + self.is_pretraining = False + + def pre_epoch_function(self, epoch, total_epochs): + pass + + @torch.no_grad() + def post_epoch_function(self, epoch, total_epochs): + if total_epochs <= 1: + self.beta.mul_(self.final_temp) + else: + self.beta.mul_(self.final_temp ** (1 / (total_epochs - 1))) + + def get_hard_mask(self, weight=None): + if self.config.pruning_parameters.enable_pruning: + return (self.s > 0).to(self.s.dtype) + return torch.tensor(1.0, device=self.s.device, dtype=self.s.dtype) + + @torch.no_grad() + def post_round_function(self): + new_s = torch.minimum(self.beta * self.s, self.s_init) + self.s.data.copy_(new_s) + self.beta.fill_(1.0) + + def calculate_additional_loss(self): + return self.config.pruning_parameters.threshold_decay * torch.norm(self.get_mask().reshape(-1), p=1) + + def get_layer_sparsity(self, weight): + return self.get_hard_mask().sum() / weight.numel() diff --git a/src/pquant/core/torch/pruning_methods/dst.py b/src/pquant/core/torch/pruning_methods/dst.py new file mode 100644 index 0000000..4cb1c5e --- /dev/null +++ b/src/pquant/core/torch/pruning_methods/dst.py @@ -0,0 +1,105 @@ +import numpy as np +import torch +import torch.nn as nn + + +def get_threshold_size(config, weight_shape): + if config.pruning_parameters.threshold_type == "layerwise": + return (1, 1) + elif config.pruning_parameters.threshold_type == "channelwise": + return (weight_shape[0], 1) + elif config.pruning_parameters.threshold_type == "weightwise": + return (weight_shape[0], int(np.prod(weight_shape[1:]))) + + +class _BinaryStep(torch.autograd.Function): + @staticmethod + def forward(ctx, weight): + ctx.save_for_backward(weight) + return (weight > 0).to(weight.dtype) + + @staticmethod + def backward(ctx, upstream): + (weight,) = ctx.saved_tensors + abs_w = weight.abs() + idx_lt04 = torch.where(abs_w <= 0.4, 2 - 4 * abs_w, torch.zeros_like(weight)) + idx_04to1 = torch.where((abs_w > 0.4) & (abs_w <= 1.0), torch.full_like(weight, 0.4), torch.zeros_like(weight)) + grads = idx_lt04 + idx_04to1 + return grads * upstream + + +def binary_step(weight): + return _BinaryStep.apply(weight) + + +class DST(nn.Module): + def __init__(self, config, layer_type, *args, **kwargs): + super().__init__() + if isinstance(config, dict): + from pquant.core.hyperparameter_optimization import PQConfig + + config = PQConfig.load_from_config(config) + self.config = config + self.layer_type = layer_type + self._is_pretraining = True + self._is_finetuning = False + self.is_pretraining = True + self.is_finetuning = False + self.built = False + + def build(self, input_shape): + if self.built: + return + threshold_size = get_threshold_size(self.config, input_shape) + self.threshold = nn.Parameter(torch.zeros(threshold_size)) + self.register_buffer("mask", torch.ones(tuple(input_shape))) + self.built = True + + def forward(self, weight): + if self._is_pretraining or self._is_finetuning: + return weight * self.mask.to(weight.dtype) + + mask = self.get_mask(weight) + ratio = 1.0 - mask.sum() / mask.numel() + if float(ratio.detach()) >= self.config.pruning_parameters.max_pruning_pct: + with torch.no_grad(): + self.threshold.data.zero_() + mask = self.get_mask(weight) + with torch.no_grad(): + self.mask.copy_(mask.detach()) + return weight * mask + + def get_hard_mask(self, weight=None): + return self.mask + + def get_mask(self, weight): + weight_orig_shape = weight.shape + weights_reshaped = weight.reshape(weight.shape[0], -1) + pre_binarystep = weights_reshaped.abs() - self.threshold + mask = binary_step(pre_binarystep) + return mask.reshape(weight_orig_shape) + + def pre_epoch_function(self, epoch, total_epochs): + pass + + def get_layer_sparsity(self, weight): + return self.get_mask(weight).sum() / weight.numel() + + def calculate_additional_loss(self): + if self._is_pretraining or self._is_finetuning: + return torch.zeros((), dtype=self.threshold.dtype, device=self.threshold.device) + return self.config.pruning_parameters.alpha * torch.sum(torch.exp(-self.threshold)) + + def pre_finetune_function(self): + self._is_finetuning = True + self.is_finetuning = True + + def post_epoch_function(self, epoch, total_epochs): + pass + + def post_pre_train_function(self): + self._is_pretraining = False + self.is_pretraining = False + + def post_round_function(self): + pass diff --git a/src/pquant/core/torch/pruning_methods/fitcompress.py b/src/pquant/core/torch/pruning_methods/fitcompress.py new file mode 100644 index 0000000..7a31d69 --- /dev/null +++ b/src/pquant/core/torch/pruning_methods/fitcompress.py @@ -0,0 +1,45 @@ +import torch +import torch.nn as nn + + +class FITCompress(nn.Module): + def __init__(self, config, *args, **kwargs): + super().__init__() + if isinstance(config, dict): + from pquant.core.hyperparameter_optimization import PQConfig + + config = PQConfig.load_from_config(config) + self.config = config + self.is_pretraining = True + self.is_finetuning = False + self.built = False + + def build(self, input_shape): + if self.built: + return + self.register_buffer("mask", torch.ones(tuple(input_shape))) + self.built = True + + def forward(self, weight): + return self.mask.to(weight.dtype) * weight + + def get_hard_mask(self, weight=None): + return self.mask + + def pre_epoch_function(self, epoch, total_epochs, **kwargs): + pass + + def calculate_additional_loss(self): + return 0.0 + + def pre_finetune_function(self): + self.is_finetuning = True + + def post_round_function(self): + pass + + def post_pre_train_function(self): + self.is_pretraining = False + + def post_epoch_function(self, epoch, total_epochs, **kwargs): + pass diff --git a/src/pquant/core/torch/pruning_methods/mdmm.py b/src/pquant/core/torch/pruning_methods/mdmm.py new file mode 100644 index 0000000..8afe615 --- /dev/null +++ b/src/pquant/core/torch/pruning_methods/mdmm.py @@ -0,0 +1,141 @@ +import inspect + +import torch +import torch.nn as nn + +from pquant.core.torch.pruning_methods.constraint_functions import ( + EqualityConstraint, + GreaterThanOrEqualConstraint, + LessThanOrEqualConstraint, +) +from pquant.core.torch.pruning_methods.metric_functions import ( + StructuredSparsityMetric, + UnstructuredSparsityMetric, +) + +_METRIC_REGISTRY = { + "UnstructuredSparsity": UnstructuredSparsityMetric, + "StructuredSparsity": StructuredSparsityMetric, +} + +_CONSTRAINT_REGISTRY = { + "Equality": EqualityConstraint, + "LessThanOrEqual": LessThanOrEqualConstraint, + "GreaterThanOrEqual": GreaterThanOrEqualConstraint, +} + + +class MDMM(nn.Module): + def __init__(self, config, layer_type, *args, **kwargs): + super().__init__() + if isinstance(config, dict): + from pquant.core.hyperparameter_optimization import PQConfig + + config = PQConfig.load_from_config(config) + self.config = config + self.layer_type = layer_type + self.constraint_layer = None + self._is_finetuning = False + self._is_pretraining = True + self.is_pretraining = True + self.is_finetuning = False + self._last_penalty = None + self.built = False + + def build(self, input_shape): + if self.built: + return + pruning_parameters = self.config.pruning_parameters + metric_type = pruning_parameters.metric_type + constraint_type = pruning_parameters.constraint_type + target_value = pruning_parameters.target_value + target_sparsity = pruning_parameters.target_sparsity + l0_mode = pruning_parameters.l0_mode + scale_mode = pruning_parameters.scale_mode + + candidate_kwargs = { + "epsilon": pruning_parameters.epsilon, + "target_sparsity": target_sparsity, + "l0_mode": l0_mode, + "scale_mode": scale_mode, + "rf": pruning_parameters.rf, + } + + metric_cls = _METRIC_REGISTRY.get(metric_type) + if metric_cls is None: + raise ValueError(f"Unknown metric_type: {metric_type}") + sig = inspect.signature(getattr(metric_cls, "__init__", metric_cls)) + metric_kwargs = {k: v for k, v in candidate_kwargs.items() if v is not None and k in sig.parameters} + metric_fn = metric_cls(**metric_kwargs) + + common_args = { + "metric_fn": metric_fn, + "target_value": target_value, + "scale": pruning_parameters.scale, + "damping": pruning_parameters.damping, + "use_grad": pruning_parameters.use_grad, + "lr": pruning_parameters.constraint_lr, + } + + constraint_type_cls = _CONSTRAINT_REGISTRY.get(constraint_type) + if constraint_type_cls is None: + raise ValueError(f"Unknown constraint_type: {constraint_type}") + self.constraint_layer = constraint_type_cls(**common_args) + + self.register_buffer("mask", torch.ones(tuple(input_shape))) + self.built = True + + def forward(self, weight): + epsilon = self.config.pruning_parameters.epsilon + hard_mask = (weight.abs() > epsilon).to(weight.dtype) + not_active = self._is_pretraining or self._is_finetuning + + if not not_active: + with torch.no_grad(): + self.mask.copy_(hard_mask.detach()) + + penalty = self.constraint_layer(weight, training=self.training).sum() + + if not_active: + self._last_penalty = torch.zeros((), device=weight.device, dtype=weight.dtype) + else: + self._last_penalty = penalty + + if self._is_finetuning: + return weight * hard_mask + return weight + + def get_hard_mask(self, weight=None): + if weight is None: + return self.mask + epsilon = self.config.pruning_parameters.epsilon + return (weight.abs() > epsilon).to(weight.dtype) + + def get_layer_sparsity(self, weight): + return self.get_hard_mask(weight).sum() / weight.numel() + + def calculate_additional_loss(self): + if self._last_penalty is None: + return 0.0 + return self._last_penalty + + def pre_epoch_function(self, epoch, total_epochs): + pass + + def pre_finetune_function(self): + self._is_finetuning = True + self.is_finetuning = True + if hasattr(self.constraint_layer, "module"): + self.constraint_layer.module.turn_off() + else: + self.constraint_layer.turn_off() + + def post_epoch_function(self, epoch, total_epochs): + pass + + def post_pre_train_function(self): + self._is_pretraining = False + self.is_pretraining = False + + def post_round_function(self): + pass diff --git a/src/pquant/core/torch/pruning_methods/metric_functions.py b/src/pquant/core/torch/pruning_methods/metric_functions.py new file mode 100644 index 0000000..e005c5c --- /dev/null +++ b/src/pquant/core/torch/pruning_methods/metric_functions.py @@ -0,0 +1,57 @@ +import torch + + +class UnstructuredSparsityMetric: + """L0-L1 based metric β€” torch port of the keras version.""" + + def __init__(self, l0_mode='coarse', scale_mode="mean", epsilon=1e-3, target_sparsity=0.8, alpha=100.0): + assert l0_mode in ['coarse', 'smooth'], "Mode must be 'coarse' or 'smooth'" + assert scale_mode in ['sum', 'mean'], "Scale mode must be 'sum' or 'mean'" + assert 0 <= target_sparsity <= 1, "target_sparsity must be between 0 and 1" + self.l0_mode = l0_mode + self.scale_mode = scale_mode + self.target_sparsity = float(target_sparsity) + self.epsilon = float(epsilon) + self.alpha = float(alpha) + self.l0_fn = self._coarse_l0 if l0_mode == 'coarse' else self._smooth_l0 + self._scaling = self._mean_scaling if scale_mode == 'mean' else self._sum_scaling + + def _sum_scaling(self, fn_value, num): + return fn_value + + def _mean_scaling(self, fn_value, num): + return fn_value / num + + def _coarse_l0(self, weight_vector): + return (weight_vector.abs() <= self.epsilon).to(torch.float32).mean() + + def _smooth_l0(self, weight_vector): + return torch.exp(-self.alpha * weight_vector.square()).mean() + + def __call__(self, weight): + num_weights = torch.tensor(float(weight.numel()), dtype=weight.dtype, device=weight.device) + flat = weight.reshape(-1) + l0_term = self.l0_fn(flat) + l1_term = flat.abs().sum() + factor = (self.target_sparsity**2) - l0_term.square() + fn_value = factor * l1_term + return self._scaling(fn_value, num_weights) + + +class StructuredSparsityMetric: + def __init__(self, rf=1, epsilon=1e-3): + self.rf = int(rf) + self.epsilon = float(epsilon) + + def __call__(self, weight): + w_reshaped = weight.reshape(weight.shape[0], -1) + num_weights = w_reshaped.shape[1] + padding = (self.rf - num_weights % self.rf) % self.rf + if padding: + w_padded = torch.nn.functional.pad(w_reshaped, (0, padding)) + else: + w_padded = w_reshaped + groups = w_padded.reshape(w_padded.shape[0], -1, self.rf) + group_norms = torch.sqrt((groups.square()).sum(dim=-1)) + zero_groups = (group_norms <= self.epsilon).to(torch.float32) + return zero_groups.sum() / float(group_norms.numel()) diff --git a/src/pquant/core/torch/pruning_methods/pdp.py b/src/pquant/core/torch/pruning_methods/pdp.py new file mode 100644 index 0000000..5204988 --- /dev/null +++ b/src/pquant/core/torch/pruning_methods/pdp.py @@ -0,0 +1,140 @@ +import math + +import torch +import torch.nn as nn + + +class PDP(nn.Module): + def __init__(self, config, layer_type, *args, **kwargs): + super().__init__() + if isinstance(config, dict): + from pquant.core.hyperparameter_optimization import PQConfig + + config = PQConfig.load_from_config(config) + self._init_r = float(config.pruning_parameters.sparsity) + self._epsilon = float(config.pruning_parameters.epsilon) + self.temp = float(config.pruning_parameters.temperature) + self.config = config + self.layer_type = layer_type + self._is_pretraining = True + self._is_finetuning = False + self.is_pretraining = True + self.is_finetuning = False + self.built = False + + # Wanda/PDP setup code externally assigns to .init_r / .sparsity β€” keep as properties so + # assignments propagate to the stored Python float used by pre_epoch_function. + @property + def init_r(self): + return self._init_r + + @init_r.setter + def init_r(self, value): + self._init_r = float(value.detach().item()) if torch.is_tensor(value) else float(value) + + def build(self, input_shape): + if self.built: + return + structured = self.config.pruning_parameters.structured_pruning + if structured: + if self.layer_type == "linear": + mask_shape = (input_shape[0], 1) + elif len(input_shape) == 3: + mask_shape = (input_shape[0], 1, 1) + else: + mask_shape = (input_shape[0], 1, 1, 1) + else: + mask_shape = tuple(input_shape) + + self.softmax_shape = list(input_shape) + [1] + self._mask_numel = math.prod(mask_shape) + self.flat_weight_size = float(self._mask_numel) + + self.register_buffer("mask", torch.ones(mask_shape)) + self.register_buffer("r", torch.tensor(self._init_r)) + + if structured: + self._compute_mask = self._mask_structured_channel if self.layer_type == "conv" else self._mask_structured_linear + else: + self._compute_mask = self._mask_unstructured + self.built = True + + def post_pre_train_function(self): + self._is_pretraining = False + self.is_pretraining = False + + @torch.no_grad() + def pre_epoch_function(self, epoch, _): + if hasattr(self, "r"): + val = min(1.0, self._epsilon * (epoch + 1)) * self._init_r + self.r.fill_(val) + + def post_round_function(self): + pass + + def pre_finetune_function(self): + self._is_finetuning = True + self.is_finetuning = True + with torch.no_grad(): + self.mask.copy_((self.mask >= 0.5).to(self.mask.dtype)) + + def post_epoch_function(self, epoch, total_epochs): + pass + + def _mask_unstructured(self, weight): + weight_reshaped = weight.reshape(self.softmax_shape) + abs_flat = weight.abs().reshape(-1) + all_vals, _ = torch.topk(abs_flat, self._mask_numel) + ind = int((1 - float(self.r.item())) * self.flat_weight_size) - 1 + lim = max(0, min(ind, int(self.flat_weight_size) - 2)) + Wh, Wt = all_vals[lim], all_vals[lim + 1] + t = torch.ones_like(weight_reshaped) * (0.5 * (Wh + Wt)) + soft_input = torch.cat((t**2, weight_reshaped**2), dim=-1) / self.temp + mw = torch.softmax(soft_input, dim=-1)[..., 1] + return mw.reshape(weight.shape) + + def _mask_structured_linear(self, weight): + norm = torch.norm(weight, dim=1, p=2, keepdim=True) + norm_flat = norm.reshape(-1) + W_all, _ = torch.topk(norm_flat, self._mask_numel) + ind = int((1 - float(self.r.item())) * self.flat_weight_size) - 1 + lim = max(0, min(ind, self._mask_numel - 2)) + Wh, Wt = W_all[lim], W_all[lim + 1] + t = torch.ones_like(norm) * 0.5 * (Wh + Wt) + soft_input = torch.cat((t**2, norm**2), dim=1) / self.temp + mw = torch.softmax(soft_input, dim=1)[..., 1] + return mw.unsqueeze(-1) + + def _mask_structured_channel(self, weight): + weight_reshaped = weight.reshape(weight.shape[0], -1) + norm = torch.norm(weight_reshaped, dim=1, p=2) + norm_flat = norm.reshape(-1) + W_all, _ = torch.topk(norm_flat, self._mask_numel) + ind = int((1 - float(self.r.item())) * self.flat_weight_size) - 1 + lim = max(0, min(ind, self._mask_numel - 2)) + Wh, Wt = W_all[lim], W_all[lim + 1] + norm = norm.unsqueeze(-1) + t = torch.ones_like(norm) * 0.5 * (Wh + Wt) + soft_input = torch.cat((t**2, norm**2), dim=-1) / self.temp + mw = torch.softmax(soft_input, dim=-1)[..., 1] + while mw.dim() < weight.dim(): + mw = mw.unsqueeze(-1) + return mw + + def forward(self, weight): + if self._is_pretraining or self._is_finetuning: + return self.mask.to(weight.dtype) * weight + new_mask = self._compute_mask(weight) + self.mask.data = new_mask + return self.mask * weight + + def get_hard_mask(self, weight=None): + return (self.mask >= 0.5).to(self.mask.dtype) + + def calculate_additional_loss(self): + return 0.0 + + def get_layer_sparsity(self, weight): + hard_mask = (self.mask >= 0.5).to(self.mask.dtype) + masked_weight = hard_mask * weight + return masked_weight.count_nonzero() / masked_weight.numel() diff --git a/src/pquant/core/torch/pruning_methods/wanda.py b/src/pquant/core/torch/pruning_methods/wanda.py new file mode 100644 index 0000000..331c65c --- /dev/null +++ b/src/pquant/core/torch/pruning_methods/wanda.py @@ -0,0 +1,160 @@ +import torch +import torch.nn as nn + + +class Wanda(nn.Module): + def __init__(self, config, layer_type, *args, **kwargs): + super().__init__() + if isinstance(config, dict): + from pquant.core.hyperparameter_optimization import PQConfig + + config = PQConfig.load_from_config(config) + self.config = config + self.act_type = "relu" + self.layer_type = layer_type + self._is_pretraining = True + self._is_finetuning = False + self.is_pretraining = True + self.is_finetuning = False + self._sparsity = float(self.config.pruning_parameters.sparsity) + self.N = self.config.pruning_parameters.N + self.M = self.config.pruning_parameters.M + self.t_start_collecting_batch = int(self.config.pruning_parameters.t_start_collecting_batch) + self.built = False + + # Expose sparsity as a tensor so it supports torch tensor API (.cpu(), etc.) + # while keeping the Python float for cheap internal arithmetic. + @property + def sparsity(self): + return torch.tensor(self._sparsity) + + @sparsity.setter + def sparsity(self, v): + self._sparsity = float(v.detach().item()) if torch.is_tensor(v) else float(v) + + def build(self, input_shape): + if self.built: + return + n_in = input_shape[0] if self.layer_type == "depthwise_conv" else input_shape[1] + self.register_buffer("mask", torch.ones(tuple(input_shape))) + self.register_buffer("inputs_sq_sum", torch.zeros(n_in)) + self.register_buffer("batches_collected", torch.zeros((), dtype=torch.int32)) + self.register_buffer("t", torch.zeros((), dtype=torch.int32)) + self.register_buffer("done", torch.zeros((), dtype=torch.bool)) + self.built = True + + @torch.no_grad() + def collect_input(self, x, weight, training): + if not training: + return + if self._is_pretraining or self._is_finetuning: + return + if bool(self.done.item()): + return + if int(self.t.item()) < self.t_start_collecting_batch: + return + + t_delta = int(self.config.pruning_parameters.t_delta) + + if self.layer_type == "linear": + per_batch_sq = (x * x).sum(dim=0) + else: + axes = (0,) + tuple(range(2, x.dim())) + per_batch_sq = (x * x).sum(dim=axes) + + self.inputs_sq_sum.add_(per_batch_sq.to(self.inputs_sq_sum.dtype)) + self.batches_collected.add_(1) + + if int(self.batches_collected.item()) == t_delta: + norm = torch.sqrt(self.inputs_sq_sum) + new_mask = self._compute_prune_mask(norm, weight) + self.mask.copy_(new_mask.to(self.mask.dtype)) + self.done.fill_(True) + self.inputs_sq_sum.zero_() + self.batches_collected.zero_() + + def _compute_prune_mask(self, norm, weight): + if self.layer_type == "linear": + return self._handle_linear(norm, weight) + if self.layer_type == "depthwise_conv": + return self._handle_depthwise_conv(norm, weight) + return self._handle_conv(norm, weight) + + def _handle_linear(self, norm, weight): + metric = weight.abs() * norm + if self.N is not None and self.M is not None: + metric_reshaped = metric.reshape(-1, self.M) + weight_reshaped = weight.reshape(-1, self.M) + mask = self.get_mask(weight_reshaped, metric_reshaped, sparsity=self.N / self.M) + return mask.reshape(weight.shape) + metric_reshaped = metric.reshape(1, -1) + weight_reshaped = weight.reshape(1, -1) + mask = self.get_mask(weight_reshaped, metric_reshaped, sparsity=self._sparsity) + return mask.reshape(weight.shape) + + def _handle_conv(self, norm, weight): + if weight.dim() == 3: + norm_reshaped = norm.reshape(1, norm.shape[0], 1) + else: + norm_reshaped = norm.reshape(1, norm.shape[0], 1, 1) + metric = weight.abs() * norm_reshaped + if self.N is not None and self.M is not None: + metric_reshaped = metric.reshape(-1, self.M) + weight_reshaped = weight.reshape(-1, self.M) + mask = self.get_mask(weight_reshaped, metric_reshaped, sparsity=self.N / self.M) + return mask.reshape(weight.shape) + metric_reshaped = metric.reshape(metric.shape[0], -1) + weight_reshaped = weight.reshape(weight.shape[0], -1) + mask = self.get_mask(weight_reshaped, metric_reshaped, sparsity=self._sparsity) + return mask.reshape(weight.shape) + + def _handle_depthwise_conv(self, norm, weight): + norm_reshaped = norm.reshape(norm.shape[0], 1, 1, 1) + metric = weight.abs() * norm_reshaped + metric_reshaped = metric.reshape(metric.shape[0], -1) + weight_reshaped = weight.reshape(weight.shape[0], -1) + mask = self.get_mask(weight_reshaped, metric_reshaped, sparsity=self._sparsity) + return mask.reshape(weight.shape) + + def get_mask(self, weight, metric, sparsity): + d0, d1 = metric.shape + keep_idxs = ( + torch.argsort(metric, dim=1, stable=True)[:, int(d1 * sparsity) :] + + torch.arange(d0, device=metric.device)[:, None] * d1 + ) + keep_idxs = keep_idxs.flatten() + kept_values = torch.zeros(weight.numel(), dtype=weight.dtype, device=weight.device) + kept_values[keep_idxs] = weight.flatten()[keep_idxs] + kept_values = kept_values.reshape(weight.shape) + return (kept_values != 0).to(weight.dtype) + + def forward(self, weight): + return self.mask.to(weight.dtype) * weight + + def get_hard_mask(self, weight=None): + return self.mask + + def post_pre_train_function(self): + self._is_pretraining = False + self.is_pretraining = False + + def pre_epoch_function(self, epoch, total_epochs, **kwargs): + pass + + def post_round_function(self): + pass + + def pre_finetune_function(self): + self._is_finetuning = True + self.is_finetuning = True + + def calculate_additional_loss(self): + return 0.0 + + def get_layer_sparsity(self, weight): + pass + + @torch.no_grad() + def post_epoch_function(self, epoch, total_epochs, **kwargs): + if not self._is_pretraining: + self.t.add_(1) diff --git a/src/pquant/core/torch/quantizer.py b/src/pquant/core/torch/quantizer.py index 7fd57f8..65e7998 100644 --- a/src/pquant/core/torch/quantizer.py +++ b/src/pquant/core/torch/quantizer.py @@ -3,7 +3,8 @@ import torch import torch.nn as nn -from pquant.core.quantizer_functions import create_quantizer +from pquant.core.torch.fixed_point_quantizer import get_fixed_quantizer +from pquant.core.torch.hgq_quantizer import HGQQuantizer class Quantizer(nn.Module): @@ -32,7 +33,9 @@ def __init__( self.i = torch.nn.Parameter(torch.tensor(i), requires_grad=False) self.f = torch.nn.Parameter(torch.tensor(f), requires_grad=False) self.b = torch.nn.Parameter(torch.tensor(i + k + f), requires_grad=False) - self.quantizer = create_quantizer(self.k, i, f, self.overflow, self.round_mode, self.use_hgq, self.is_data, place) + self.quantizer = create_quantizer( + self.k, i, f, self.overflow, self.round_mode, self.use_hgq, self.is_data, gamma=hgq_gamma + ) self.is_pretraining = True self.hgq_gamma = hgq_gamma if isinstance(granularity, Enum): @@ -45,7 +48,7 @@ def __init__( def get_quantization_bits(self): if self.use_hgq: - return self.quantizer.quantizer.k, self.quantizer.quantizer.i, self.quantizer.quantizer.f + return self.quantizer.k, self.quantizer.i, self.quantizer.f else: return self.k, self.i, self.f @@ -58,8 +61,7 @@ def get_total_bits(self, shape): def set_quantization_bits(self, i, f): if self.use_hgq: - self.quantizer.quantizer._i.assign(self.quantizer.quantizer._i * 0.0 + i) - self.quantizer.quantizer._f.assign(self.quantizer.quantizer._f * 0.0 + f) + self.quantizer.set_bits(i, f) self.i.data = torch.tensor(i) self.f.data = torch.tensor(f) @@ -108,21 +110,20 @@ def forward(self, x): def hgq_loss(self): if self.is_pretraining or not self.use_hgq: return 0.0 - loss = 0 - for layer_loss in self.quantizer.quantizer.losses: - loss += layer_loss - return loss + return self.quantizer.regularization_loss() def post_epoch_function(self): - if self.use_hgq and self.quantizer.quantizer.built: - constrained_i = self.quantizer.quantizer._i.constraint(self.quantizer.quantizer._i) - self.quantizer.quantizer._i.assign(constrained_i) - constrained_f = self.quantizer.quantizer._f.constraint(self.quantizer.quantizer._f) - self.quantizer.quantizer._f.assign(constrained_f) + if self.use_hgq and self.quantizer.built: + self.quantizer.post_epoch_constraint_apply() def apply_final_compression(self): if self.use_hgq and not self.quantizer.built: return + if self.use_hgq: + with torch.no_grad(): + self.quantizer._f.data.clamp_(self.quantizer.f_min, self.quantizer.f_max) + if self.quantizer.overflow_mode != "WRAP": + self.quantizer._i.data.clamp_(self.quantizer.i_min, self.quantizer.i_max) _, i, f = self.get_quantization_bits() self.i.data = i self.f.data = f @@ -140,5 +141,14 @@ def initialize_quantization_parameters(self, i, f): def reload_from_local(self): if not self.use_hgq: return - self.quantizer.quantizer._i.assign(self.i) - self.quantizer.quantizer._f.assign(self.f) + self.quantizer.set_bits(self.i, self.f) + + +def create_quantizer(k, i, f, overflow, round_mode, is_heterogeneous, is_data, gamma=1e-8): + if is_heterogeneous: + if is_data: + return HGQQuantizer(k0=k, i0=i, f0=f, overflow_mode=overflow, round_mode=round_mode, is_data=True, gamma=gamma) + else: + return HGQQuantizer(k0=k, i0=i, f0=f, overflow_mode=overflow, round_mode=round_mode, is_data=False, gamma=gamma) + else: + return get_fixed_quantizer(round_mode=round_mode, overflow_mode=overflow) diff --git a/src/pquant/core/torch/utils.py b/src/pquant/core/torch/utils.py new file mode 100644 index 0000000..8f0f0d7 --- /dev/null +++ b/src/pquant/core/torch/utils.py @@ -0,0 +1,25 @@ +from pquant.core.torch.pruning_methods.activation_pruning import ActivationPruning +from pquant.core.torch.pruning_methods.autosparse import AutoSparse +from pquant.core.torch.pruning_methods.cs import ContinuousSparsification +from pquant.core.torch.pruning_methods.dst import DST +from pquant.core.torch.pruning_methods.mdmm import MDMM +from pquant.core.torch.pruning_methods.pdp import PDP +from pquant.core.torch.pruning_methods.wanda import Wanda + + +def get_pruning_layer(config, layer_type): + pruning_method = config.pruning_parameters.pruning_method + if pruning_method == "dst": + return DST(config, layer_type) + elif pruning_method == "autosparse": + return AutoSparse(config, layer_type) + elif pruning_method == "cs": + return ContinuousSparsification(config, layer_type) + elif pruning_method == "pdp": + return PDP(config, layer_type) + elif pruning_method == "activation_pruning": + return ActivationPruning(config, layer_type) + elif pruning_method == "wanda": + return Wanda(config, layer_type) + elif pruning_method == "mdmm": + return MDMM(config, layer_type) diff --git a/src/pquant/core/utils.py b/src/pquant/core/utils.py index fe9e575..072e7bd 100644 --- a/src/pquant/core/utils.py +++ b/src/pquant/core/utils.py @@ -2,35 +2,6 @@ import yaml -from pquant.pruning_methods.activation_pruning import ActivationPruning -from pquant.pruning_methods.autosparse import AutoSparse -from pquant.pruning_methods.cs import ContinuousSparsification -from pquant.pruning_methods.dst import DST -from pquant.pruning_methods.fitcompress import FITCompress -from pquant.pruning_methods.mdmm import MDMM -from pquant.pruning_methods.pdp import PDP -from pquant.pruning_methods.wanda import Wanda - - -def get_pruning_layer(config, layer_type): - pruning_method = config.pruning_parameters.pruning_method - if pruning_method == "dst": - return DST(config, layer_type) - elif pruning_method == "autosparse": - return AutoSparse(config, layer_type) - elif pruning_method == "cs": - return ContinuousSparsification(config, layer_type) - elif pruning_method == "pdp": - return PDP(config, layer_type) - elif pruning_method == "activation_pruning": - return ActivationPruning(config, layer_type) - elif pruning_method == "wanda": - return Wanda(config, layer_type) - elif pruning_method == "mdmm": - return MDMM(config, layer_type) - elif pruning_method == "fitcompress": - return FITCompress(config) - def get_default_config(pruning_method: str): assert pruning_method in [ @@ -38,7 +9,6 @@ def get_default_config(pruning_method: str): "ap", "cs", "dst", - "fitcompress", "pdp", "wanda", "mdmm", diff --git a/src/pquant/pruning_methods/fitcompress.py b/src/pquant/pruning_methods/fitcompress.py deleted file mode 100644 index f2b3b5c..0000000 --- a/src/pquant/pruning_methods/fitcompress.py +++ /dev/null @@ -1,52 +0,0 @@ -import keras - - -@keras.saving.register_keras_serializable(package="Layers") -class FITCompress(keras.layers.Layer): - def __init__(self, config, *args, **kwargs): - super().__init__(*args, **kwargs) - if isinstance(config, dict): - from pquant.core.hyperparameter_optimization import PQConfig - - config = PQConfig.load_from_config(config) - self.config = config - self.is_pretraining = True - self.is_finetuning = False - - def build(self, input_shape): - self.mask = self.add_weight(shape=input_shape, initializer="ones", trainable=False) - super().build(input_shape) - - def call(self, weight): - return self.mask * weight - - def get_hard_mask(self, weight=None): - return self.mask - - def pre_epoch_function(self, epoch, total_epochs): - pass - - def calculate_additional_loss(*args, **kwargs): - return 0 - - def pre_finetune_function(self): - self.is_finetuning = True - - def post_round_function(self): - pass - - def post_pre_train_function(self): - self.is_pretraining = False - - def post_epoch_function(self, epoch, total_epochs): - pass - - def get_config(self): - config = super().get_config() - - config.update( - { - "config": self.config.get_dict(), - } - ) - return config diff --git a/tests/test_ap.py b/tests/test_ap.py index d126898..afd4d9c 100644 --- a/tests/test_ap.py +++ b/tests/test_ap.py @@ -2,7 +2,7 @@ from keras import ops from keras.random import shuffle -from pquant.pruning_methods.activation_pruning import ActivationPruning +from pquant.core.keras.pruning_methods.activation_pruning import ActivationPruning @pytest.fixture diff --git a/tests/test_hgq_torch.py b/tests/test_hgq_torch.py new file mode 100644 index 0000000..9166725 --- /dev/null +++ b/tests/test_hgq_torch.py @@ -0,0 +1,250 @@ +""" +Parity tests: PyTorch `HGQQuantizer` vs Keras `hgq.Quantizer` (hgq2 library). + +Both implementations should produce matching forward outputs, matching +gradients on the fractional-bit parameter `f`, and should follow a similar +training trajectory when fed the same data with the same initial state. +""" + +import os + +os.environ.setdefault("KERAS_BACKEND", "torch") + +import pytest # noqa: E402 +import torch # noqa: E402 + +hgq = pytest.importorskip("hgq") +from hgq.quantizer import Quantizer as KerasHGQQuantizer # noqa: E402 +from hgq.quantizer import QuantizerConfig # noqa: E402 + +from pquant.core.torch.hgq_quantizer import HGQQuantizer # noqa: E402 + +RTOL = 1e-4 +ATOL = 1e-5 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_keras_q(k, i, f, overflow, round_mode, is_data, place="datalane"): + """Construct the Keras hgq2 quantizer mirroring create_hgq_*_quantizer().""" + homogeneous_axis = (0,) if is_data else () + cfg = QuantizerConfig( + q_type="kif", + place="datalane" if is_data else place, + k0=k, + i0=i, + f0=f, + overflow_mode=overflow, + round_mode=round_mode, + homogeneous_axis=homogeneous_axis, + ) + return KerasHGQQuantizer(config=cfg) + + +def _as_torch(x): + """Unwrap a keras tensor produced with the torch backend to a torch.Tensor.""" + if isinstance(x, torch.Tensor): + return x + import keras.ops as ops + + return ops.convert_to_tensor(x) + + +# --------------------------------------------------------------------------- +# Forward parity +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "overflow,round_mode,is_data", + [ + ("SAT", "RND", False), + ("SAT", "RND_CONV", False), + ("SAT_SYM", "RND", False), + ("WRAP", "RND", False), + ("WRAP", "RND", True), + ("SAT", "RND", True), + ], +) +def test_forward_matches_keras(overflow, round_mode, is_data): + torch.manual_seed(0) + k, i, f = 1, 2, 4 + x = torch.randn(8, 5) * 0.8 + + torch_q = HGQQuantizer( + k0=k, + i0=i, + f0=f, + overflow_mode=overflow, + round_mode=round_mode, + is_data=is_data, + ) + keras_q = _make_keras_q(k, i, f, overflow, round_mode, is_data) + + out_torch = torch_q(x, training=False).detach() + # Keras quantizer: call with training=False + out_keras = _as_torch(keras_q(x, training=False)).detach() + + assert out_torch.shape == out_keras.shape, f"shape mismatch: {out_torch.shape} vs {out_keras.shape}" + assert torch.allclose(out_torch, out_keras, rtol=RTOL, atol=ATOL), ( + f"[{overflow}/{round_mode}/is_data={is_data}] " f"max diff = {(out_torch - out_keras).abs().max().item():.6g}" + ) + + +def test_forward_matches_keras_training_sat(): + torch.manual_seed(1) + x = torch.randn(4, 6) * 1.2 + torch_q = HGQQuantizer(k0=1, i0=1, f0=3, overflow_mode="SAT", round_mode="RND", is_data=False) + keras_q = _make_keras_q(1, 1, 3, "SAT", "RND", is_data=False) + + out_torch = torch_q(x, training=True).detach() + out_keras = _as_torch(keras_q(x, training=True)).detach() + + assert torch.allclose(out_torch, out_keras, rtol=RTOL, atol=ATOL), ( + f"training forward diverged, max diff = " f"{(out_torch - out_keras).abs().max().item():.6g}" + ) + + +# --------------------------------------------------------------------------- +# Backward parity β€” gradient through f +# --------------------------------------------------------------------------- + + +def test_backward_f_gradient_sat(): + torch.manual_seed(2) + x_data = torch.randn(4, 5) * 0.6 + + torch_q = HGQQuantizer(k0=1, i0=2, f0=4, overflow_mode="SAT", round_mode="RND", is_data=False) + keras_q = _make_keras_q(1, 2, 4, "SAT", "RND", is_data=False) + + # Forward to trigger build for both + x_t = x_data.clone().requires_grad_(True) + out_t = torch_q(x_t, training=True) + loss_t = out_t.pow(2).sum() + loss_t.backward() + grad_f_torch = torch_q._f.grad.detach().clone() + + x_k = x_data.clone().requires_grad_(True) + out_k = _as_torch(keras_q(x_k, training=True)) + loss_k = out_k.pow(2).sum() + loss_k.backward() + + # hgq2 stores the f parameter as keras_q.quantizer._f β€” access via state + keras_f = _find_f_param(keras_q) + grad_f_keras = keras_f.grad.detach().clone() if keras_f.grad is not None else None + + assert grad_f_keras is not None, "Keras hgq quantizer produced no gradient on _f" + assert grad_f_torch.shape == grad_f_keras.shape, f"grad f shape mismatch: {grad_f_torch.shape} vs {grad_f_keras.shape}" + # Gradient direction/magnitude should match up to STE discretisation noise. + assert torch.allclose(grad_f_torch, grad_f_keras, rtol=1e-3, atol=1e-5), ( + f"grad f mismatch, max diff = " f"{(grad_f_torch - grad_f_keras).abs().max().item():.6g}" + ) + + +def test_backward_input_gradient_ste(): + """STE: gradient w.r.t. input should be approximately identity within sat range.""" + torch.manual_seed(3) + x = (torch.randn(3, 4) * 0.3).requires_grad_(True) # small β†’ within sat bounds + + torch_q = HGQQuantizer(k0=1, i0=2, f0=4, overflow_mode="SAT", round_mode="RND", is_data=False) + out = torch_q(x, training=True) + out.sum().backward() + + assert x.grad is not None + assert torch.allclose(x.grad, torch.ones_like(x), atol=1e-5), ( + f"STE grad should be ~1 inside sat range, got max deviation " f"{(x.grad - 1).abs().max().item():.6g}" + ) + + +# --------------------------------------------------------------------------- +# Training trajectory parity +# --------------------------------------------------------------------------- + + +def test_training_trajectory_matches(): + """Run several SGD steps on both implementations with same data/init. The + learned f parameter should stay close (exact equality not guaranteed due to + STE non-determinism in rounding ties, but trajectories should track).""" + torch.manual_seed(4) + k, i, f = 1, 2, 4 + lr = 0.05 + steps = 20 + + torch_q = HGQQuantizer(k0=k, i0=i, f0=f, overflow_mode="SAT", round_mode="RND", is_data=False) + keras_q = _make_keras_q(k, i, f, "SAT", "RND", is_data=False) + + # Prime build with a dummy pass + dummy = torch.zeros(4, 3) + torch_q(dummy, training=False) + keras_q(dummy, training=False) + + # Optimizer: plain SGD on f (and i for SAT) for both + torch_opt = torch.optim.SGD([p for p in torch_q.parameters() if p.requires_grad], lr=lr) + keras_f = _find_f_param(keras_q) + keras_i = _find_i_param(keras_q) + keras_params = [p for p in (keras_f, keras_i) if p is not None and p.requires_grad] + keras_opt = torch.optim.SGD(keras_params, lr=lr) + + for step in range(steps): + torch.manual_seed(100 + step) + x = torch.randn(4, 3) * 0.5 + + torch_opt.zero_grad() + loss_t = torch_q(x, training=True).pow(2).sum() + loss_t.backward() + torch_opt.step() + + keras_opt.zero_grad() + loss_k = _as_torch(keras_q(x, training=True)).pow(2).sum() + loss_k.backward() + keras_opt.step() + + # After training, the learned f should match within reasonable tolerance. + diff_f = (torch_q._f.detach() - keras_f.detach()).abs().max().item() + assert diff_f < 0.25, f"f parameters diverged after {steps} steps: max diff = {diff_f:.4f}" + + # Forward outputs on a held-out batch should also be close. + torch.manual_seed(999) + x_test = torch.randn(4, 3) * 0.5 + out_t = torch_q(x_test, training=False).detach() + out_k = _as_torch(keras_q(x_test, training=False)).detach() + diff_out = (out_t - out_k).abs().max().item() + assert diff_out < 0.1, f"trained outputs diverged: max diff = {diff_out:.4f}" + + +# --------------------------------------------------------------------------- +# Utility: locate f / i parameters inside the Keras hgq2 quantizer +# --------------------------------------------------------------------------- + + +def _find_f_param(keras_q): + """Locate the fractional-bit parameter on the Keras hgq2 quantizer. + + hgq2 stores it as `keras_q.quantizer._f` (Keras Variable, which the torch + backend exposes as a torch.nn.Parameter). + """ + inner = getattr(keras_q, "quantizer", keras_q) + for name in ("_f", "f"): + p = getattr(inner, name, None) + if p is None: + continue + # Keras Variables expose a torch `.value` parameter under the torch backend + val = getattr(p, "value", p) + if isinstance(val, torch.nn.Parameter) or isinstance(val, torch.Tensor): + return val + raise RuntimeError("Could not locate f parameter on Keras hgq quantizer") + + +def _find_i_param(keras_q): + inner = getattr(keras_q, "quantizer", keras_q) + for name in ("_i", "i"): + p = getattr(inner, name, None) + if p is None: + continue + val = getattr(p, "value", p) + if isinstance(val, torch.nn.Parameter) or isinstance(val, torch.Tensor): + return val + return None diff --git a/tests/test_pdp.py b/tests/test_pdp.py index 4a6a081..3a1ec1c 100644 --- a/tests/test_pdp.py +++ b/tests/test_pdp.py @@ -2,7 +2,7 @@ from keras import ops from keras.random import shuffle -from pquant.pruning_methods.pdp import PDP +from pquant.core.keras.pruning_methods.pdp import PDP @pytest.fixture diff --git a/tests/test_torch_compression_layers.py b/tests/test_torch_compression_layers.py index cb4d1ff..3f2f80c 100644 --- a/tests/test_torch_compression_layers.py +++ b/tests/test_torch_compression_layers.py @@ -606,8 +606,8 @@ def test_hgq_weight_shape(config_pdp, dense_input): model = add_compression_layers(model, config_pdp, dense_input.shape) post_pretrain_functions(model, config_pdp) - assert model.submodule.weight_quantizer.quantizer.quantizer._i.shape == model.submodule.weight.shape - assert model.activation.input_quantizer.quantizer.quantizer._i.shape == (1, OUT_FEATURES) + assert model.submodule.weight_quantizer.quantizer._i.shape == model.submodule.weight.shape + assert model.activation.input_quantizer.quantizer._i.shape == (1, OUT_FEATURES) def test_qbn_build(config_pdp, conv2d_input): @@ -619,7 +619,7 @@ def test_qbn_build(config_pdp, conv2d_input): model = add_compression_layers(model, config_pdp, conv2d_input.shape) post_pretrain_functions(model, config_pdp) - assert model.submodule.weight_quantizer.quantizer.quantizer._i.shape == model.submodule.weight.shape + assert model.submodule.weight_quantizer.quantizer._i.shape == model.submodule.weight.shape def test_set_activation_custom_bits_hgq(config_pdp, conv2d_input): @@ -634,13 +634,13 @@ def test_set_activation_custom_bits_hgq(config_pdp, conv2d_input): if isinstance(m, (PQWeightBiasBase)): assert m.i_weight == 0.0 assert m.i_bias == 0.0 - assert torch.all(m.weight_quantizer.quantizer.quantizer.i == 0.0) - assert torch.all(m.weight_quantizer.quantizer.quantizer.i == 0.0) + assert torch.all(m.weight_quantizer.quantizer.i == 0.0) + assert torch.all(m.weight_quantizer.quantizer.i == 0.0) assert m.f_weight == 7.0 assert m.f_bias == 7.0 - assert torch.all(m.weight_quantizer.quantizer.quantizer.f == 7.0) - assert torch.all(m.weight_quantizer.quantizer.quantizer.f == 7.0) + assert torch.all(m.weight_quantizer.quantizer.f == 7.0) + assert torch.all(m.weight_quantizer.quantizer.f == 7.0) elif isinstance(m, PQActivation) and m.activation_name == "tanh": k_input, i_input, f_input = m.get_input_quantization_bits() @@ -655,8 +655,8 @@ def test_set_activation_custom_bits_hgq(config_pdp, conv2d_input): elif isinstance(m, PQAvgPool2d): assert m.i_input == 0.0 assert m.f_input == 7.0 - assert torch.all(m.input_quantizer.quantizer.quantizer.i == 0.0) - assert torch.all(m.input_quantizer.quantizer.quantizer.f == 7.0) + assert torch.all(m.input_quantizer.quantizer.i == 0.0) + assert torch.all(m.input_quantizer.quantizer.f == 7.0) config_pdp.quantization_parameters.layer_specific = { 'submodule': { @@ -675,13 +675,13 @@ def test_set_activation_custom_bits_hgq(config_pdp, conv2d_input): if isinstance(m, (PQWeightBiasBase)): assert m.i_weight == 1.0 assert m.i_bias == 2.0 - assert torch.all(m.weight_quantizer.quantizer.quantizer.i == 1.0) - assert torch.all(m.bias_quantizer.quantizer.quantizer.i == 2.0) + assert torch.all(m.weight_quantizer.quantizer.i == 1.0) + assert torch.all(m.bias_quantizer.quantizer.i == 2.0) assert m.f_weight == 3.0 assert m.f_bias == 4.0 - assert torch.all(m.weight_quantizer.quantizer.quantizer.f == 3.0) - assert torch.all(m.bias_quantizer.quantizer.quantizer.f == 4.0) + assert torch.all(m.weight_quantizer.quantizer.f == 3.0) + assert torch.all(m.bias_quantizer.quantizer.f == 4.0) elif isinstance(m, PQActivation) and m.activation_name == "tanh": k_input, i_input, f_input = m.get_input_quantization_bits() @@ -695,8 +695,8 @@ def test_set_activation_custom_bits_hgq(config_pdp, conv2d_input): elif isinstance(m, PQAvgPool2d): assert m.i_input == 1.0 assert m.f_input == 3.0 - assert torch.all(m.input_quantizer.quantizer.quantizer.i == 1.0) - assert torch.all(m.input_quantizer.quantizer.quantizer.f == 3.0) + assert torch.all(m.input_quantizer.quantizer.i == 1.0) + assert torch.all(m.input_quantizer.quantizer.f == 3.0) def test_disable_pruning_from_single_layer(config_pdp, conv2d_input): diff --git a/tests/test_torch_pruning_layers.py b/tests/test_torch_pruning_layers.py new file mode 100644 index 0000000..495dd15 --- /dev/null +++ b/tests/test_torch_pruning_layers.py @@ -0,0 +1,626 @@ +""" +Parity tests: PyTorch pruning methods vs Keras reference. + +For each pruning layer (Activation, PDP, CS, DST, Wanda, AutoSparse, MDMM) +the torch implementation must produce numerically matching outputs and +masks when given identical state and inputs as the keras version. Metric +functions (Structured/Unstructured sparsity) are also compared directly. +""" + +import os + +# The keras layers are expected to run on the tensorflow backend (the long-term +# target for the keras implementation). Run this test file with +# ``KERAS_BACKEND=tensorflow`` so the keras forward paths (including those +# routed through ``ops.custom_gradient``) exercise their intended backend. +os.environ.setdefault("KERAS_BACKEND", "tensorflow") + +import numpy as np # noqa: E402 +import pytest # noqa: E402 +import torch # noqa: E402 +from keras import ops # noqa: E402 + +from pquant.core.keras.pruning_methods.activation_pruning import ( # noqa: E402 + ActivationPruning as KActivationPruning, +) +from pquant.core.keras.pruning_methods.autosparse import ( # noqa: E402 + AutoSparse as KAutoSparse, +) +from pquant.core.keras.pruning_methods.cs import ( # noqa: E402 + ContinuousSparsification as KCS, +) +from pquant.core.keras.pruning_methods.dst import DST as KDST # noqa: E402 +from pquant.core.keras.pruning_methods.mdmm import MDMM as KMDMM # noqa: E402 +from pquant.core.keras.pruning_methods.metric_functions import ( # noqa: E402 + StructuredSparsityMetric as KStructuredSparsityMetric, +) +from pquant.core.keras.pruning_methods.metric_functions import ( # noqa: E402 + UnstructuredSparsityMetric as KUnstructuredSparsityMetric, +) +from pquant.core.keras.pruning_methods.pdp import PDP as KPDP # noqa: E402 +from pquant.core.keras.pruning_methods.wanda import Wanda as KWanda # noqa: E402 +from pquant.core.torch.pruning_methods.activation_pruning import ( # noqa: E402 + ActivationPruning as TActivationPruning, +) +from pquant.core.torch.pruning_methods.autosparse import ( # noqa: E402 + AutoSparse as TAutoSparse, +) +from pquant.core.torch.pruning_methods.cs import ( # noqa: E402 + ContinuousSparsification as TCS, +) +from pquant.core.torch.pruning_methods.dst import DST as TDST # noqa: E402 +from pquant.core.torch.pruning_methods.mdmm import MDMM as TMDMM # noqa: E402 +from pquant.core.torch.pruning_methods.metric_functions import ( # noqa: E402 + StructuredSparsityMetric as TStructuredSparsityMetric, +) +from pquant.core.torch.pruning_methods.metric_functions import ( # noqa: E402 + UnstructuredSparsityMetric as TUnstructuredSparsityMetric, +) +from pquant.core.torch.pruning_methods.pdp import PDP as TPDP # noqa: E402 +from pquant.core.torch.pruning_methods.wanda import Wanda as TWanda # noqa: E402 + +ATOL = 1e-5 +RTOL = 1e-4 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _to_numpy(x): + if isinstance(x, torch.Tensor): + return x.detach().cpu().numpy() + return np.asarray(ops.convert_to_numpy(x)) + + +def _assert_close(a, b, atol=ATOL, rtol=RTOL, msg=""): + a_np = _to_numpy(a) + b_np = _to_numpy(b) + assert a_np.shape == b_np.shape, f"{msg}: shape mismatch: {a_np.shape} vs {b_np.shape}" + np.testing.assert_allclose(a_np, b_np, atol=atol, rtol=rtol, err_msg=msg) + + +def _keras_tensor(arr): + return ops.convert_to_tensor(np.asarray(arr).astype(np.float32)) + + +def _torch_tensor(arr): + return torch.as_tensor(np.asarray(arr).astype(np.float32)) + + +def _reset_seed(seed=0): + np.random.seed(seed) + torch.manual_seed(seed) + + +# --------------------------------------------------------------------------- +# ActivationPruning +# --------------------------------------------------------------------------- + + +def _ap_config(): + return { + "pruning_parameters": { + "pruning_method": "activation_pruning", + "disable_pruning_for_layers": [], + "enable_pruning": True, + "threshold": 0.3, + "t_start_collecting_batch": 0, + "threshold_decay": 0.0, + "t_delta": 2, + }, + } + + +@pytest.mark.parametrize( + "layer_type,shape", + [ + ("linear", (16, 8)), + ("conv", (16, 8, 3, 3)), + ], +) +def test_activation_pruning_matches_keras(layer_type, shape): + cfg = _ap_config() + out_channels = shape[0] + batch = 32 + + _reset_seed() + weight_np = np.random.randn(*shape).astype(np.float32) + # Construct outputs with distinct per-channel activity levels so the + # resulting mask is non-trivial (some channels pct_active > threshold, + # some below). linspace from -0.5 to 1 puts roughly 1/3 of the channels + # below zero, guaranteeing those get pruned while the rest survive. + per_channel = np.linspace(-0.5, 1.0, num=out_channels, dtype=np.float32) + np.random.shuffle(per_channel) + if layer_type == "linear": + output_np = np.tile(per_channel[None, :], (batch, 1)) + else: + output_np = np.tile(per_channel[None, :, None, None], (batch, 1, 4, 4)) + + k = KActivationPruning(cfg, layer_type) + k.build(shape) + k.post_pre_train_function() + + t = TActivationPruning(cfg, layer_type) + t.build(shape) + t.post_pre_train_function() + + for _ in range(cfg["pruning_parameters"]["t_delta"]): + k.collect_output(_keras_tensor(output_np), training=True) + t.collect_output(_torch_tensor(output_np), training=True) + + k.post_epoch_function(0, 1) + t.post_epoch_function(0, 1) + + _assert_close(k.mask, t.mask, msg=f"AP mask ({layer_type})") + + # Sanity check: the constructed per-channel outputs put ~1/3 of the + # channels at non-positive values, so their pct_active == 0 falls below + # the 0.3 threshold and they should be pruned. Channels with a positive + # value hit pct_active == 1 and survive. The expected pruned count is + # deterministic from per_channel β€” verifying we actually exercise both + # branches rather than matching a trivial all-ones mask. + mask_np = _to_numpy(t.mask) + pruned_fraction = float((mask_np == 0).sum()) / mask_np.size + # linspace(-0.5, 1.0, 16) β†’ 6 values <= 0 β†’ 6/16 = 0.375 pruned channels. + assert pruned_fraction == pytest.approx(0.375), f"AP mask ({layer_type}) pruned fraction {pruned_fraction} != 0.375" + + k_out = k(_keras_tensor(weight_np)) + t_out = t(_torch_tensor(weight_np)) + _assert_close(k_out, t_out, msg=f"AP forward ({layer_type})") + + +# --------------------------------------------------------------------------- +# PDP +# --------------------------------------------------------------------------- + + +def _pdp_config(sparsity=0.75, structured=False): + return { + "pruning_parameters": { + "pruning_method": "pdp", + "disable_pruning_for_layers": [], + "enable_pruning": True, + "epsilon": 1.0, + "sparsity": sparsity, + "temperature": 1e-5, + "threshold_decay": 0.0, + "structured_pruning": structured, + }, + } + + +@pytest.mark.parametrize( + "layer_type,shape,structured", + [ + ("linear", (16, 8), False), + ("linear", (16, 8), True), + ("conv", (16, 8, 3, 3), False), + ("conv", (16, 8, 3, 3), True), + ], +) +def test_pdp_matches_keras(layer_type, shape, structured): + cfg = _pdp_config(structured=structured) + target_sparsity = cfg["pruning_parameters"]["sparsity"] + + _reset_seed() + weight_np = np.random.randn(*shape).astype(np.float32) + + k = KPDP(cfg, layer_type) + k.build(shape) + k.post_pre_train_function() + + t = TPDP(cfg, layer_type) + t.build(shape) + t.post_pre_train_function() + + # Force the sparsity ramp to be fully complete. pre_epoch_function sets + # r = min(1, epsilon * (epoch + 1)) * init_r; with epsilon=1.0 that already + # puts the ramp multiplier at 1.0 on epoch 0, so r = init_r = 0.75. + k.pre_epoch_function(0, None) + t.pre_epoch_function(0, None) + + k_out = k(_keras_tensor(weight_np)) + t_out = t(_torch_tensor(weight_np)) + _assert_close(k_out, t_out, msg=f"PDP forward ({layer_type}, structured={structured})") + + k.update_mask(_keras_tensor(weight_np)) + t.update_mask(_torch_tensor(weight_np)) + _assert_close(k.mask, t.mask, msg="PDP mask after update_mask") + + # Verify the produced mask hits the configured target sparsity. + # For structured pruning the mask has shape (C, 1, ...) and encodes + # per-channel keep/prune; its sparsity directly equals the channel-level + # pruning fraction. For unstructured it's per-element. With temperature + # 1e-5 the soft mask is effectively binary, so use >= 0.5 to discretize. + t_mask_np = _to_numpy(t.mask) + actual_sparsity = float((t_mask_np < 0.5).sum()) / t_mask_np.size + assert actual_sparsity == pytest.approx(target_sparsity, abs=1e-6), ( + f"PDP {layer_type} (structured={structured}) mask sparsity " f"{actual_sparsity} != target {target_sparsity}" + ) + + +# --------------------------------------------------------------------------- +# ContinuousSparsification +# --------------------------------------------------------------------------- + + +def _cs_config(threshold_decay=1e-4): + return { + "pruning_parameters": { + "pruning_method": "cs", + "disable_pruning_for_layers": [], + "enable_pruning": True, + "threshold_init": -0.1, + "final_temp": 200, + "threshold_decay": threshold_decay, + }, + } + + +@pytest.mark.parametrize( + "shape", + [(16, 8), (16, 8, 3, 3)], +) +def test_cs_matches_keras(shape): + cfg = _cs_config() + layer_type = "linear" if len(shape) == 2 else "conv" + + _reset_seed() + s_override_np = (np.random.randn(*shape) * 0.5).astype(np.float32) + weight_np = np.random.randn(*shape).astype(np.float32) + + k = KCS(cfg, layer_type) + k.build(shape) + k.post_pre_train_function() + k.s.assign(_keras_tensor(s_override_np)) + + t = TCS(cfg, layer_type) + t.build(shape) + t.post_pre_train_function() + with torch.no_grad(): + t.s.data.copy_(_torch_tensor(s_override_np)) + + k_out = k(_keras_tensor(weight_np)) + t_out = t(_torch_tensor(weight_np)) + _assert_close(k_out, t_out, msg=f"CS forward ({layer_type})") + + _assert_close(k.get_hard_mask(), t.get_hard_mask(), msg="CS hard mask") + _assert_close(k.calculate_additional_loss(), t.calculate_additional_loss(), msg="CS additional loss") + + # post_epoch_function updates beta β€” trajectories should match. + k.post_epoch_function(0, 5) + t.post_epoch_function(0, 5) + _assert_close(k.beta, t.beta, msg="CS beta after post_epoch_function") + + +# --------------------------------------------------------------------------- +# DST +# --------------------------------------------------------------------------- + + +def _dst_config(threshold_type="channelwise"): + return { + "pruning_parameters": { + "pruning_method": "dst", + "disable_pruning_for_layers": [], + "enable_pruning": True, + "alpha": 5e-6, + "max_pruning_pct": 0.99, + "threshold_init": 0.0, + "threshold_type": threshold_type, + "threshold_decay": 0.0, + }, + } + + +@pytest.mark.parametrize( + "layer_type,shape,threshold_type", + [ + ("linear", (16, 8), "layerwise"), + ("linear", (16, 8), "channelwise"), + ("linear", (16, 8), "weightwise"), + ("conv", (16, 8, 3, 3), "channelwise"), + ], +) +def test_dst_matches_keras(layer_type, shape, threshold_type): + cfg = _dst_config(threshold_type=threshold_type) + + _reset_seed() + weight_np = (np.random.randn(*shape) * 0.5).astype(np.float32) + if threshold_type == "layerwise": + thr_np = np.array([[0.1]], dtype=np.float32) + elif threshold_type == "channelwise": + thr_np = (np.random.rand(shape[0], 1) * 0.2).astype(np.float32) + else: # weightwise + thr_np = (np.random.rand(shape[0], int(np.prod(shape[1:]))) * 0.2).astype(np.float32) + + k = KDST(cfg, layer_type) + k.build(shape) + k.post_pre_train_function() + k.threshold.assign(_keras_tensor(thr_np)) + + t = TDST(cfg, layer_type) + t.build(shape) + t.post_pre_train_function() + with torch.no_grad(): + t.threshold.data.copy_(_torch_tensor(thr_np)) + + k_out = k(_keras_tensor(weight_np)) + t_out = t(_torch_tensor(weight_np)) + _assert_close(k_out, t_out, msg=f"DST forward ({layer_type}, {threshold_type})") + + _assert_close( + k.get_mask(_keras_tensor(weight_np)), + t.get_mask(_torch_tensor(weight_np)), + msg=f"DST get_mask ({threshold_type})", + ) + _assert_close(k.calculate_additional_loss(), t.calculate_additional_loss(), msg="DST additional loss") + + +# --------------------------------------------------------------------------- +# Wanda +# --------------------------------------------------------------------------- + + +def _wanda_config(sparsity=0.75, N=None, M=None): + return { + "pruning_parameters": { + "pruning_method": "wanda", + "disable_pruning_for_layers": [], + "enable_pruning": True, + "sparsity": sparsity, + "t_delta": 2, + "t_start_collecting_batch": 0, + "N": N, + "M": M, + "threshold_decay": 0.0, + "calculate_pruning_budget": True, + }, + } + + +@pytest.mark.parametrize( + "layer_type,shape,N,M", + [ + ("linear", (16, 8), None, None), + ("conv", (16, 8, 3, 3), None, None), + ("linear", (4, 8), 4, 8), + ("conv", (4, 8, 3, 3), 4, 8), + ], +) +def test_wanda_matches_keras(layer_type, shape, N, M): + cfg = _wanda_config(N=N, M=M) + + _reset_seed() + if layer_type == "linear": + x_np = np.random.randn(32, shape[1]).astype(np.float32) + else: + x_np = np.random.randn(32, shape[1], shape[2], shape[3]).astype(np.float32) + w_np = np.random.randn(*shape).astype(np.float32) + + k = KWanda(cfg, layer_type) + k.build(shape) + k.post_pre_train_function() + + t = TWanda(cfg, layer_type) + t.build(shape) + t.post_pre_train_function() + + for _ in range(cfg["pruning_parameters"]["t_delta"]): + k.collect_input(_keras_tensor(x_np), _keras_tensor(w_np), training=True) + t.collect_input(_torch_tensor(x_np), _torch_tensor(w_np), training=True) + + _assert_close(k.mask, t.mask, msg=f"Wanda mask ({layer_type}, N={N}, M={M})") + + # Verify the mask hits the configured target sparsity. For N:M pruning + # Wanda internally uses N/M as the sparsity target; for unstructured it + # uses the configured sparsity directly. Mask values are strictly {0, 1} + # (produced by topk + scatter), so `== 0` counts pruned entries. + target_sparsity = (N / M) if (N is not None and M is not None) else cfg["pruning_parameters"]["sparsity"] + mask_np = _to_numpy(t.mask) + pruned_fraction = float((mask_np == 0).sum()) / mask_np.size + assert pruned_fraction == pytest.approx( + target_sparsity + ), f"Wanda {layer_type} (N={N}, M={M}) pruned fraction {pruned_fraction} != target {target_sparsity}" + + k_out = k(_keras_tensor(w_np)) + t_out = t(_torch_tensor(w_np)) + _assert_close(k_out, t_out, msg=f"Wanda forward ({layer_type}, N={N}, M={M})") + + +# --------------------------------------------------------------------------- +# AutoSparse +# --------------------------------------------------------------------------- + + +def _autosparse_config(threshold_type="channelwise", threshold_init=-2.0): + return { + "pruning_parameters": { + "pruning_method": "autosparse", + "disable_pruning_for_layers": [], + "enable_pruning": True, + "alpha": 0.5, + "alpha_reset_epoch": 100, + "autotune_epochs": 10, + "backward_sparsity": False, + "threshold_init": threshold_init, + "threshold_type": threshold_type, + "threshold_decay": 0.0, + }, + } + + +@pytest.mark.parametrize( + "layer_type,shape,threshold_type", + [ + ("linear", (16, 8), "layerwise"), + ("linear", (16, 8), "channelwise"), + ("conv", (16, 8, 3, 3), "channelwise"), + ], +) +def test_autosparse_matches_keras(layer_type, shape, threshold_type): + # Keras AutoSparse.call() unconditionally routes through ops.custom_gradient, + # which under the torch backend errors because self.alpha is a keras Variable + # (save_for_backward rejects it). The path works fine under the tensorflow + # backend, which is the intended target for the keras implementation, so we + # skip this test when keras is on torch to avoid spurious failures. + import keras as _keras + + if _keras.backend.backend() == "torch": + pytest.skip("Keras AutoSparse forward is incompatible with the torch backend.") + cfg = _autosparse_config(threshold_type=threshold_type) + + _reset_seed() + weight_np = np.random.randn(*shape).astype(np.float32) + + k = KAutoSparse(cfg, layer_type) + k.build(shape) + k.post_pre_train_function() + + t = TAutoSparse(cfg, layer_type) + t.build(shape) + t.post_pre_train_function() + with torch.no_grad(): + t.threshold.data.copy_(_torch_tensor(_to_numpy(k.threshold))) + + _assert_close( + k.get_mask(_keras_tensor(weight_np)), + t.get_mask(_torch_tensor(weight_np)), + msg=f"AutoSparse get_mask ({layer_type}, {threshold_type})", + ) + + k_out = k(_keras_tensor(weight_np)) + t_out = t(_torch_tensor(weight_np)) + _assert_close(k_out, t_out, msg=f"AutoSparse forward ({layer_type}, {threshold_type})") + + # post_epoch_function updates alpha via decay; trajectories should match. + k.post_epoch_function(3, 10) + t.post_epoch_function(3, 10) + _assert_close(k.alpha, t.alpha, msg="AutoSparse alpha after post_epoch_function") + + +# --------------------------------------------------------------------------- +# MDMM +# --------------------------------------------------------------------------- + + +def _mdmm_config( + constraint_type="Equality", + metric_type="UnstructuredSparsity", + target_value=0.5, + use_grad=True, +): + return { + "pruning_parameters": { + "pruning_method": "mdmm", + "disable_pruning_for_layers": [], + "enable_pruning": True, + "constraint_type": constraint_type, + "target_value": target_value, + "metric_type": metric_type, + "target_sparsity": 0.8, + "rf": 1, + "epsilon": 1e-3, + "scale": 1.0, + "damping": 1.0, + "use_grad": use_grad, + "l0_mode": "coarse", + "scale_mode": "mean", + "constraint_lr": 1e-3, + "threshold_decay": 0.0, + }, + } + + +@pytest.mark.parametrize( + "constraint_type,metric_type", + [ + ("Equality", "UnstructuredSparsity"), + ("LessThanOrEqual", "UnstructuredSparsity"), + ("GreaterThanOrEqual", "UnstructuredSparsity"), + ("Equality", "StructuredSparsity"), + ], +) +def test_mdmm_matches_keras(constraint_type, metric_type): + cfg = _mdmm_config(constraint_type=constraint_type, metric_type=metric_type) + shape = (16, 8) + + _reset_seed() + weight_np = (np.random.randn(*shape) * 0.2).astype(np.float32) + + k = KMDMM(cfg, "linear") + k.build(shape) + k.post_pre_train_function() + + t = TMDMM(cfg, "linear") + t.build(shape) + t.post_pre_train_function() + + k_out = k(_keras_tensor(weight_np)) + t_out = t(_torch_tensor(weight_np)) + _assert_close(k_out, t_out, msg=f"MDMM forward ({constraint_type}, {metric_type})") + + _assert_close( + k.get_hard_mask(_keras_tensor(weight_np)), + t.get_hard_mask(_torch_tensor(weight_np)), + msg="MDMM hard_mask", + ) + + # Constraint penalty: read directly from the constraint layer to avoid + # differences in how keras/torch surface accumulated losses. + k_penalty = ops.sum(k.constraint_layer(_keras_tensor(weight_np))) + t_penalty = t.constraint_layer(_torch_tensor(weight_np)).sum() + _assert_close(k_penalty, t_penalty, msg="MDMM constraint penalty") + + +def test_mdmm_finetune_returns_masked_weight(): + """In finetuning mode both layers should return weight * hard_mask.""" + cfg = _mdmm_config() + shape = (8, 6) + + _reset_seed() + weight_np = (np.random.randn(*shape) * 0.2).astype(np.float32) + + k = KMDMM(cfg, "linear") + k.build(shape) + k.post_pre_train_function() + k.pre_finetune_function() + + t = TMDMM(cfg, "linear") + t.build(shape) + t.post_pre_train_function() + t.pre_finetune_function() + + k_out = k(_keras_tensor(weight_np)) + t_out = t(_torch_tensor(weight_np)) + _assert_close(k_out, t_out, msg="MDMM finetune forward") + + +# --------------------------------------------------------------------------- +# Metric functions +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("l0_mode", ["coarse", "smooth"]) +@pytest.mark.parametrize("scale_mode", ["mean", "sum"]) +def test_unstructured_sparsity_metric_matches_keras(l0_mode, scale_mode): + k = KUnstructuredSparsityMetric(l0_mode=l0_mode, scale_mode=scale_mode, target_sparsity=0.7, epsilon=1e-3) + t = TUnstructuredSparsityMetric(l0_mode=l0_mode, scale_mode=scale_mode, target_sparsity=0.7, epsilon=1e-3) + + _reset_seed() + w_np = (np.random.randn(16, 8) * 0.1).astype(np.float32) + + _assert_close(k(_keras_tensor(w_np)), t(_torch_tensor(w_np)), msg=f"Unstructured({l0_mode},{scale_mode})") + + +@pytest.mark.parametrize("rf", [1, 4, 5]) +def test_structured_sparsity_metric_matches_keras(rf): + k = KStructuredSparsityMetric(rf=rf, epsilon=1e-3) + t = TStructuredSparsityMetric(rf=rf, epsilon=1e-3) + + _reset_seed() + w_np = (np.random.randn(12, 7) * 0.05).astype(np.float32) + + _assert_close(k(_keras_tensor(w_np)), t(_torch_tensor(w_np)), msg=f"Structured(rf={rf})") diff --git a/tests/test_wanda.py b/tests/test_wanda.py index 609572d..3af1026 100644 --- a/tests/test_wanda.py +++ b/tests/test_wanda.py @@ -2,7 +2,7 @@ import pytest from keras import ops -from pquant.pruning_methods.wanda import Wanda +from pquant.core.keras.pruning_methods.wanda import Wanda @pytest.fixture From 734ab81dec88b162a4bf5e167e4e0e37fe3fbf52 Mon Sep 17 00:00:00 2001 From: nroope Date: Mon, 8 Jun 2026 13:18:13 +0200 Subject: [PATCH 07/22] Distiller and onnx converter and pqmha (#40) * initial onnx converter,layerwise and model distillation for torch, mha * gelu, leakyrelu activations , layernorm for Torch * dynamic data quantization option --- src/pquant/configs/config_ap.yaml | 1 + src/pquant/configs/config_autosparse.yaml | 1 + src/pquant/configs/config_cs.yaml | 1 + src/pquant/configs/config_dst.yaml | 1 + src/pquant/configs/config_fitcompress.yaml | 1 + src/pquant/configs/config_mdmm.yaml | 1 + src/pquant/configs/config_pdp.yaml | 1 + src/pquant/configs/config_wanda.yaml | 1 + src/pquant/configs/finetuning.yaml | 1 + src/pquant/core/keras/activations.py | 3 + src/pquant/core/keras/convert_to_onnx.py | 1511 ++++++++++++++ src/pquant/core/keras/layers.py | 277 +++ src/pquant/core/keras/quantizer.py | 41 +- src/pquant/core/torch/activations.py | 12 +- src/pquant/core/torch/convert_to_onnx.py | 1865 ++++++++++++++++++ src/pquant/core/torch/distillers.py | 686 +++++++ src/pquant/core/torch/layers.py | 497 ++++- src/pquant/core/torch/quantizer.py | 47 +- src/pquant/data_models/quantization_model.py | 1 + tests/run_tests.sh | 2 + tests/test_keras_onnx_converter.py | 197 ++ tests/test_torch_compression_layers.py | 17 +- tests/test_torch_onnx_converter.py | 384 ++++ 23 files changed, 5505 insertions(+), 44 deletions(-) create mode 100644 src/pquant/core/keras/convert_to_onnx.py create mode 100644 src/pquant/core/torch/convert_to_onnx.py create mode 100644 src/pquant/core/torch/distillers.py create mode 100644 tests/test_keras_onnx_converter.py create mode 100644 tests/test_torch_onnx_converter.py diff --git a/src/pquant/configs/config_ap.yaml b/src/pquant/configs/config_ap.yaml index 0c3cb22..729ed2d 100644 --- a/src/pquant/configs/config_ap.yaml +++ b/src/pquant/configs/config_ap.yaml @@ -13,6 +13,7 @@ quantization_parameters: default_data_keep_negatives: 0. default_data_integer_bits: 0. default_data_fractional_bits: 7. + dynamic_data_quantization: false granularity: "per_tensor" quantize_input: true quantize_output: false diff --git a/src/pquant/configs/config_autosparse.yaml b/src/pquant/configs/config_autosparse.yaml index e5ce8ed..7ac7ffd 100644 --- a/src/pquant/configs/config_autosparse.yaml +++ b/src/pquant/configs/config_autosparse.yaml @@ -16,6 +16,7 @@ quantization_parameters: default_data_keep_negatives: 0. default_data_integer_bits: 0. default_data_fractional_bits: 7. + dynamic_data_quantization: false granularity: "per_tensor" quantize_input: true quantize_output: false diff --git a/src/pquant/configs/config_cs.yaml b/src/pquant/configs/config_cs.yaml index 88594c8..58bbdad 100644 --- a/src/pquant/configs/config_cs.yaml +++ b/src/pquant/configs/config_cs.yaml @@ -12,6 +12,7 @@ quantization_parameters: default_data_keep_negatives: 0. default_data_integer_bits: 0. default_data_fractional_bits: 7. + dynamic_data_quantization: false granularity: "per_tensor" quantize_input: true quantize_output: false diff --git a/src/pquant/configs/config_dst.yaml b/src/pquant/configs/config_dst.yaml index 9a54073..86b958a 100644 --- a/src/pquant/configs/config_dst.yaml +++ b/src/pquant/configs/config_dst.yaml @@ -14,6 +14,7 @@ quantization_parameters: default_data_keep_negatives: 0. default_data_integer_bits: 0. default_data_fractional_bits: 7. + dynamic_data_quantization: false granularity: "per_tensor" quantize_input: true quantize_output: false diff --git a/src/pquant/configs/config_fitcompress.yaml b/src/pquant/configs/config_fitcompress.yaml index cd8e502..e18b8e8 100644 --- a/src/pquant/configs/config_fitcompress.yaml +++ b/src/pquant/configs/config_fitcompress.yaml @@ -11,6 +11,7 @@ quantization_parameters: default_data_keep_negatives: 0. default_data_integer_bits: 0. default_data_fractional_bits: 7. + dynamic_data_quantization: false granularity: "per_tensor" quantize_input: true quantize_output: false diff --git a/src/pquant/configs/config_mdmm.yaml b/src/pquant/configs/config_mdmm.yaml index abbae3c..7586882 100644 --- a/src/pquant/configs/config_mdmm.yaml +++ b/src/pquant/configs/config_mdmm.yaml @@ -24,6 +24,7 @@ quantization_parameters: default_data_keep_negatives: 0. default_data_integer_bits: 0. default_data_fractional_bits: 7. + dynamic_data_quantization: false granularity: "per_tensor" quantize_input: true quantize_output: false diff --git a/src/pquant/configs/config_pdp.yaml b/src/pquant/configs/config_pdp.yaml index 937b5b9..1461d76 100644 --- a/src/pquant/configs/config_pdp.yaml +++ b/src/pquant/configs/config_pdp.yaml @@ -14,6 +14,7 @@ quantization_parameters: default_data_keep_negatives: 0. default_data_integer_bits: 0. default_data_fractional_bits: 7. + dynamic_data_quantization: false granularity: "per_tensor" quantize_input: true quantize_output: false diff --git a/src/pquant/configs/config_wanda.yaml b/src/pquant/configs/config_wanda.yaml index f128994..ab9cdae 100644 --- a/src/pquant/configs/config_wanda.yaml +++ b/src/pquant/configs/config_wanda.yaml @@ -16,6 +16,7 @@ quantization_parameters: default_data_keep_negatives: 0. default_data_integer_bits: 0. default_data_fractional_bits: 7. + dynamic_data_quantization: false granularity: "per_tensor" quantize_input: true quantize_output: false diff --git a/src/pquant/configs/finetuning.yaml b/src/pquant/configs/finetuning.yaml index 70a6237..165aef5 100644 --- a/src/pquant/configs/finetuning.yaml +++ b/src/pquant/configs/finetuning.yaml @@ -13,6 +13,7 @@ quantization_parameters: integer_bits: 4 fractional_bits: 6 use_high_granularity_quantization: false + dynamic_data_quantization: false granularity: "per_tensor" use_real_tanh: false use_symmetric_quantization: false diff --git a/src/pquant/core/keras/activations.py b/src/pquant/core/keras/activations.py index dcf5f6b..5ae4542 100644 --- a/src/pquant/core/keras/activations.py +++ b/src/pquant/core/keras/activations.py @@ -68,6 +68,7 @@ def __init__( self.hgq_gamma = config.quantization_parameters.hgq_gamma self.hgq_heterogeneous = config.quantization_parameters.hgq_heterogeneous self.use_fitcompress = config.fitcompress_parameters.enable_fitcompress + self.dynamic_data = config.quantization_parameters.dynamic_data_quantization self.post_fitcompress_calibration = False self.saved_inputs = [] @@ -89,6 +90,7 @@ def build(self, input_shape): is_heterogeneous=self.use_hgq, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.dynamic_data, ) if self.quantize_output: self.output_quantizer = Quantizer( @@ -101,6 +103,7 @@ def build(self, input_shape): is_heterogeneous=self.use_hgq, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.dynamic_data, ) if self.use_multiplier: diff --git a/src/pquant/core/keras/convert_to_onnx.py b/src/pquant/core/keras/convert_to_onnx.py new file mode 100644 index 0000000..c35e9f4 --- /dev/null +++ b/src/pquant/core/keras/convert_to_onnx.py @@ -0,0 +1,1511 @@ +""" +Convert a PQuant Keras functional model to ONNX or QONNX format. + +Pass ``use_qonnx=True`` to emit QONNX ``Quant`` custom nodes (requires the +qonnx runtime). Pass ``use_qonnx=False`` (default) to emit standard +``Clip + QuantizeLinear + DequantizeLinear`` nodes runnable with plain +onnxruntime. + +Fixed-point (k, i, f) mapping +------------------------------ +QONNX: + scale = 2^(-f) + zero_point = 0 + bit_width = k + i + f + signed = int(k) + +Standard ONNX (QDQ): + scale = 2^(-f) + zero_point = 0 (int8 signed, uint8 unsigned) + clip range = [-2^i, 2^i - 2^(-f)] signed + = [0, 2^i - 2^(-f)] unsigned + +Keras weight layout (kernel always stored as HWIO regardless of data_format): + Conv2D kernel: [kH, kW, in/g, out] β†’ [out, in/g, kH, kW] for ONNX + Conv1D kernel: [kL, in/g, out] β†’ [out, in/g, kL] for ONNX + DepthwiseConv2D kernel: [kH, kW, in, dm] β†’ [in*dm, 1, kH, kW] for ONNX + Dense kernel: [in, out] β†’ stored as [out, in] for Gemm (transB=1) + +Data format: + channels_first Data flows as NCHW; Conv/Pool ONNX ops work naturally. + channels_last Transpose(NHWCβ†’NCHW) is inserted before each Conv/Pool/BN + node and Transpose(NCHWβ†’NHWC) is inserted after. The + logical data format in the ONNX graph therefore stays NHWC + at every inter-layer edge; only the PQ ops run internally in + NCHW. ONNX-aware optimisers (e.g. onnxsim) can fold the + redundant back-to-back transposes away. +""" + +import functools +import logging + +import keras +import numpy as np +import onnx +import onnx.helper as oh +import onnx.numpy_helper as onh +from keras import ops +from onnx import TensorProto + +from pquant.core.keras.activations import PQActivation +from pquant.core.keras.layers import ( + PQBatchNormalization, + PQConv1d, + PQConv2d, + PQDense, + PQDepthwiseConv2d, + PQMultiheadAttention, +) + +# --------------------------------------------------------------------------- +# QONNX Quant node +# --------------------------------------------------------------------------- + +ROUND_MODE_MAP = { + "TRN": "FLOOR", + "RND": "ROUND", + "RND_CONV": "ROUND", + "TRN_ZERO": "TRUNCATE", + "RND_ZERO": "ROUND", + "RND_MIN_INF": "FLOOR", + "RND_INF": "ROUND", +} + + +def _quant_node(name_prefix, input_name, rounding_mode, k, i, f, initializers, overflow_mode="SAT"): + """Build a QONNX Quant node. k/i/f are numpy arrays. Returns ([node], output_name).""" + k_val = int(float(np.array(k).ravel()[0])) + f_arr = np.array(f, dtype=np.float32) + i_arr = np.array(i, dtype=np.float32) + if f_arr.size > 1: + i_arr = i_arr.ravel().max() + f_arr = f_arr.ravel().min() + i_val = float(i_arr) + f_val = float(f_arr) + scale = float(2.0 ** (-f_val)) + bit_width = float(k_val + i_val + f_val) + qonnx_rnd = ROUND_MODE_MAP.get(rounding_mode, "ROUND") + # SAT_SYM excludes the most-negative value β†’ QONNX narrow=1 + narrow = 1 if (k_val == 1 and overflow_mode == "SAT_SYM") else 0 + + scale_name = f"{name_prefix}_scale" + zp_name = f"{name_prefix}_zero_point" + bw_name = f"{name_prefix}_bit_width" + out_name = f"{name_prefix}_quantized" + + initializers.append(onh.from_array(np.array(scale, dtype=np.float32), name=scale_name)) + initializers.append(onh.from_array(np.array(0.0, dtype=np.float32), name=zp_name)) + initializers.append(onh.from_array(np.array(bit_width, dtype=np.float32), name=bw_name)) + + node = oh.make_node( + op_type="Quant", + inputs=[input_name, scale_name, zp_name, bw_name], + outputs=[out_name], + domain="qonnx.custom_op.general", + signed=k_val, + narrow=narrow, + rounding_mode=qonnx_rnd, + ) + return [node], out_name + + +# --------------------------------------------------------------------------- +# Standard ONNX QDQ triple +# --------------------------------------------------------------------------- + + +def _qdq_node( + name_prefix, input_name, rounding_mode, k, i, f, initializers, overflow_mode="SAT", include_clip=True +): # noqa: ARG001 + """Build QuantizeLinear+DequantizeLinear nodes, optionally preceded by a Clip. + + Returns ([nodes], output_name). Set include_clip=False to skip the Clip node + (safe when values are guaranteed to be in-range at inference time, since + QuantizeLinear saturates naturally). + """ + k_val = int(float(np.array(k).ravel()[0])) + i_val = float(np.array(i, dtype=np.float32).ravel()[0]) + f_val = float(np.array(f, dtype=np.float32).ravel()[0]) + scale = float(2.0 ** (-f_val)) + signed = k_val == 1 + + clip_max = float(2.0**i_val - 2.0 ** (-f_val)) + if not signed: + clip_min = 0.0 + elif overflow_mode == "SAT_SYM": + clip_min = -clip_max # symmetric: -(2^i - 2^(-f)) + else: + clip_min = float(-(2.0**i_val)) # SAT: -2^i + zp_val = np.int8(0) if signed else np.uint8(0) + + scale_name = f"{name_prefix}_scale" + zp_name = f"{name_prefix}_zero_point" + quantized_name = f"{name_prefix}_quantized" + out_name = f"{name_prefix}_dequantized" + + initializers += [ + onh.from_array(np.array(scale, dtype=np.float32), name=scale_name), + onh.from_array(np.array(zp_val), name=zp_name), + ] + + if include_clip: + clip_min_name = f"{name_prefix}_clip_min" + clip_max_name = f"{name_prefix}_clip_max" + clipped_name = f"{name_prefix}_clipped" + initializers += [ + onh.from_array(np.array(clip_min, dtype=np.float32), name=clip_min_name), + onh.from_array(np.array(clip_max, dtype=np.float32), name=clip_max_name), + ] + nodes = [ + oh.make_node("Clip", inputs=[input_name, clip_min_name, clip_max_name], outputs=[clipped_name]), + oh.make_node("QuantizeLinear", inputs=[clipped_name, scale_name, zp_name], outputs=[quantized_name]), + ] + else: + nodes = [ + oh.make_node("QuantizeLinear", inputs=[input_name, scale_name, zp_name], outputs=[quantized_name]), + ] + + nodes.append(oh.make_node("DequantizeLinear", inputs=[quantized_name, scale_name, zp_name], outputs=[out_name])) + return nodes, out_name + + +# --------------------------------------------------------------------------- +# integer weight storage helper +# --------------------------------------------------------------------------- + + +def _int_weight_node(name_prefix, weight_np, k, i, f, initializers): # noqa: ARG001 (i unused) + """ + Store a weight tensor as int8/uint8 + DequantizeLinear. + + weight_np must already be in ONNX layout (transposed from Keras) and on the + fixed-point grid after apply_final_compression(). + + k/i/f are numpy arrays (may be per-tensor scalar or per-channel 1-D after + caller has already squeezed/reshaped appropriately). + + Granularity: + - per-tensor (f is scalar): single scale. + - per-channel (f is 1-D of length out_channels): axis=0 on weight tensor. + - per-weight (fully per-element): falls back to float32 storage. + + Returns ([node], output_name). + """ + k_np = np.array(k, dtype=np.float32) + f_np = np.array(f, dtype=np.float32) + k_val = int(float(k_np.ravel()[0])) + dtype = np.int8 if k_val == 1 else np.uint8 + out_channels = weight_np.shape[0] + out_name = f"{name_prefix}_dequantized" + + if f_np.size == 1: + # per-tensor + scale_np = np.array(float(2.0 ** (-float(f_np.ravel()[0]))), dtype=np.float32) + int_w = np.round(weight_np / float(scale_np)).astype(dtype) + per_ch = False + else: + f_1d = f_np.ravel() + if f_1d.size == out_channels: + # per-channel: one f value per output channel + scale_1d = (2.0 ** (-f_1d)).astype(np.float32) + bcast = scale_1d.reshape((out_channels,) + (1,) * (weight_np.ndim - 1)) + int_w = np.round(weight_np / bcast).astype(dtype) + scale_np = scale_1d + per_ch = True + else: + # per-weight: ONNX cannot represent; fall back to float32 + float_name = f"{name_prefix}_float" + initializers.append(onh.from_array(weight_np, name=float_name)) + return [], float_name + + int_name = f"{name_prefix}_int" + scale_name = f"{name_prefix}_dq_scale" + zp_name = f"{name_prefix}_dq_zp" + + zp_np = np.zeros(out_channels if per_ch else 1, dtype=dtype) + initializers += [ + onh.from_array(int_w, name=int_name), + onh.from_array(scale_np, name=scale_name), + onh.from_array(zp_np if per_ch else np.array(dtype(0)), name=zp_name), + ] + node_kwargs = {"axis": 0} if per_ch else {} + node = oh.make_node("DequantizeLinear", inputs=[int_name, scale_name, zp_name], outputs=[out_name], **node_kwargs) + return [node], out_name + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + + +def _np(tensor): + """Convert a Keras tensor / variable / scalar to a float32 numpy array.""" + return np.array(tensor, dtype=np.float32) + + +def _bn_transpose_info(layer): + """ + Return (need_transpose, perm_fwd, perm_bwd) for a BatchNormalization layer. + + ONNX BN (opset < 14) always normalises on axis 1 (NCHW). If the Keras + layer uses axis=-1 (channels_last), we must insert Transpose nodes around + the BN op. We infer ndim from the layer's stored input_shape. + """ + axis = getattr(layer, "axis", 1) + stored = getattr(layer, "input_shape", None) + ndim = len(stored) if stored is not None else 4 + eff_axis = axis if axis >= 0 else (ndim + axis) + + if eff_axis == 1 or ndim <= 2: + # channels already at position 1, or 2-D input β€” no transpose needed + return False, None, None + + if ndim == 4 and eff_axis == 3: + return True, [0, 3, 1, 2], [0, 2, 3, 1] + + if ndim == 3 and eff_axis == 2: + return True, [0, 2, 1], [0, 2, 1] + + # Fallback: general permutation that moves eff_axis to position 1 + perm_fwd = [0, eff_axis] + [i for i in range(1, ndim) if i != eff_axis] + # Inverse permutation + perm_bwd = [0] * ndim + for i, p in enumerate(perm_fwd): + perm_bwd[p] = i + return True, perm_fwd, perm_bwd + + +def _maybe_quant_input(layer, prefix, current, nodes, initializers, quant_fn): + if ( + getattr(layer, "input_quantizer", None) is not None + and getattr(layer, "quantize_input", True) + and getattr(layer, "enable_quantization", True) + ): + q = layer.input_quantizer + k, i, f = q.get_quantization_bits() + new_nodes, current = quant_fn( + f"{prefix}_in", + current, + q.round_mode, + _np(k), + _np(i), + _np(f), + initializers, + overflow_mode=getattr(q, "overflow", "SAT"), + ) + nodes.extend(new_nodes) + return current + + +def _maybe_quant_output(layer, prefix, current, nodes, initializers, quant_fn): + if ( + getattr(layer, "output_quantizer", None) is not None + and getattr(layer, "quantize_output", False) + and getattr(layer, "enable_quantization", True) + ): + q = layer.output_quantizer + k, i, f = q.get_quantization_bits() + new_nodes, current = quant_fn( + f"{prefix}_out", + current, + q.round_mode, + _np(k), + _np(i), + _np(f), + initializers, + overflow_mode=getattr(q, "overflow", "SAT"), + ) + nodes.extend(new_nodes) + return current + + +def _add_transpose(name, input_name, perm, nodes): + """Emit a Transpose node and return the output name.""" + out = f"{name}_transpose_{''.join(str(p) for p in perm)}" + nodes.append(oh.make_node("Transpose", inputs=[input_name], outputs=[out], perm=list(perm))) + return out + + +def _channels_last(layer): + return getattr(layer, "data_format", "channels_first") == "channels_last" + + +def _weight_f_for_onnx(f_np, out_channels): + """Squeeze/ravel a Keras per-channel f array to shape (out_channels,) for ONNX.""" + f_flat = f_np.ravel() + if f_flat.size == 1: + return f_flat # scalar, return as-is + if f_flat.size == out_channels: + return f_flat + # Per-element or mismatched: take the minimum to avoid overflow + return np.array([f_flat.min()], dtype=np.float32) + + +# --------------------------------------------------------------------------- +# per-layer graph builders +# --------------------------------------------------------------------------- + + +def _add_dense(layer, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights): + """Dense / PQDense. Keras kernel [in, out] stored as [out, in]; Gemm uses transB=1. + + Storing the weight pre-transposed means axis=0 is always the output dimension, + which is required for per-channel DequantizeLinear and avoids a runtime Transpose. + """ + current = _maybe_quant_input(layer, prefix, current, nodes, initializers, quant_fn) + + # Transpose kernel to [out, in]; Gemm will use transB=1 so Y = X @ W^T = X @ kernel. + kernel_np = _np(layer._kernel).T # [out, in] + out_units = kernel_np.shape[0] + + if use_qonnx: + k_w, i_w, f_w = layer.weight_quantizer.get_quantization_bits() + wfp_name = f"{prefix}_weight_fp" + initializers.append(onh.from_array(kernel_np, name=wfp_name)) + w_nodes, q_weight = _quant_node( + f"{prefix}_weight", + wfp_name, + layer.weight_quantizer.round_mode, + _np(k_w), + _np(i_w), + _np(f_w), + initializers, + overflow_mode=getattr(layer.weight_quantizer, "overflow", "SAT"), + ) + nodes.extend(w_nodes) + elif store_integer_weights: + k_w, i_w, f_w = layer.weight_quantizer.get_quantization_bits() + f_np_w = _np(f_w) + f_for_onnx = _weight_f_for_onnx(f_np_w, out_units) + k_for_onnx = _weight_f_for_onnx(_np(k_w), out_units) + i_for_onnx = _weight_f_for_onnx(_np(i_w), out_units) + w_nodes, q_weight = _int_weight_node(f"{prefix}_weight", kernel_np, k_for_onnx, i_for_onnx, f_for_onnx, initializers) + nodes.extend(w_nodes) + else: + q_weight = f"{prefix}_weight" + initializers.append(onh.from_array(kernel_np, name=q_weight)) + + gemm_inputs = [current, q_weight] + + if layer._bias is not None: + bias_np = _np(layer._bias) + if use_qonnx: + k_b, i_b, f_b = layer.bias_quantizer.get_quantization_bits() + bfp_name = f"{prefix}_bias_fp" + initializers.append(onh.from_array(bias_np, name=bfp_name)) + b_nodes, q_bias = _quant_node( + f"{prefix}_bias", + bfp_name, + layer.bias_quantizer.round_mode, + _np(k_b), + _np(i_b), + _np(f_b), + initializers, + overflow_mode=getattr(layer.bias_quantizer, "overflow", "SAT"), + ) + nodes.extend(b_nodes) + elif store_integer_weights: + k_b, i_b, f_b = layer.bias_quantizer.get_quantization_bits() + b_nodes, q_bias = _int_weight_node(f"{prefix}_bias", bias_np, _np(k_b), _np(i_b), _np(f_b), initializers) + nodes.extend(b_nodes) + else: + q_bias = f"{prefix}_bias" + initializers.append(onh.from_array(bias_np, name=q_bias)) + gemm_inputs.append(q_bias) + + gemm_out = f"{prefix}_gemm" + nodes.append(oh.make_node("Gemm", inputs=gemm_inputs, outputs=[gemm_out], transB=1)) + current = gemm_out + + current = _maybe_quant_output(layer, prefix, current, nodes, initializers, quant_fn) + return current + + +def _add_conv(layer, prefix, current, nodes, initializers, ndim, quant_fn, use_qonnx, store_integer_weights): + """PQConv2d / PQConv1d. Keras kernel: [*kernel, in/g, out] β†’ ONNX [out, in/g, *kernel].""" + cl = _channels_last(layer) + + if cl: + perm_to_nchw = [0, 3, 1, 2] if ndim == 2 else [0, 2, 1] + perm_to_nhwx = [0, 2, 3, 1] if ndim == 2 else [0, 2, 1] + current = _add_transpose(f"{prefix}_pre", current, perm_to_nchw, nodes) + + current = _maybe_quant_input(layer, prefix, current, nodes, initializers, quant_fn) + + kernel_np = _np(layer._kernel) + # Transpose kernel from Keras HWIO to ONNX OIHW + if ndim == 2: + kernel_onnx = np.transpose(kernel_np, (3, 2, 0, 1)) # [kH,kW,in,out] β†’ [out,in,kH,kW] + else: + kernel_onnx = np.transpose(kernel_np, (2, 1, 0)) # [kL,in,out] β†’ [out,in,kL] + + out_channels = kernel_onnx.shape[0] + + if use_qonnx: + k_w, i_w, f_w = layer.weight_quantizer.get_quantization_bits() + wfp_name = f"{prefix}_weight_fp" + initializers.append(onh.from_array(kernel_onnx, name=wfp_name)) + w_nodes, q_weight = _quant_node( + f"{prefix}_weight", + wfp_name, + layer.weight_quantizer.round_mode, + _np(k_w), + _np(i_w), + _np(f_w), + initializers, + overflow_mode=getattr(layer.weight_quantizer, "overflow", "SAT"), + ) + nodes.extend(w_nodes) + elif store_integer_weights: + k_w, i_w, f_w = layer.weight_quantizer.get_quantization_bits() + f_for_onnx = _weight_f_for_onnx(_np(f_w), out_channels) + k_for_onnx = _weight_f_for_onnx(_np(k_w), out_channels) + i_for_onnx = _weight_f_for_onnx(_np(i_w), out_channels) + w_nodes, q_weight = _int_weight_node( + f"{prefix}_weight", kernel_onnx, k_for_onnx, i_for_onnx, f_for_onnx, initializers + ) + nodes.extend(w_nodes) + else: + q_weight = f"{prefix}_weight" + initializers.append(onh.from_array(kernel_onnx, name=q_weight)) + + conv_inputs = [current, q_weight] + + if layer._bias is not None: + bias_np = _np(layer._bias) + if use_qonnx: + k_b, i_b, f_b = layer.bias_quantizer.get_quantization_bits() + bfp_name = f"{prefix}_bias_fp" + initializers.append(onh.from_array(bias_np, name=bfp_name)) + b_nodes, q_bias = _quant_node( + f"{prefix}_bias", + bfp_name, + layer.bias_quantizer.round_mode, + _np(k_b), + _np(i_b), + _np(f_b), + initializers, + overflow_mode=getattr(layer.bias_quantizer, "overflow", "SAT"), + ) + nodes.extend(b_nodes) + elif store_integer_weights: + k_b, i_b, f_b = layer.bias_quantizer.get_quantization_bits() + b_nodes, q_bias = _int_weight_node(f"{prefix}_bias", bias_np, _np(k_b), _np(i_b), _np(f_b), initializers) + nodes.extend(b_nodes) + else: + q_bias = f"{prefix}_bias" + initializers.append(onh.from_array(bias_np, name=q_bias)) + conv_inputs.append(q_bias) + + # Padding + padding = layer.padding + if isinstance(padding, str): + auto_pad = "SAME_UPPER" if padding == "same" else "VALID" + pads = None + else: + p = list(padding) if hasattr(padding, "__iter__") else [padding] * ndim + pads = p + p # ONNX format: [begin_0, begin_1, ..., end_0, end_1, ...] + auto_pad = "NOTSET" + + to_list = lambda v, n: list(v) if hasattr(v, "__iter__") else [v] * n # noqa: E731 + conv_attrs = dict( + kernel_shape=to_list(layer.kernel_size, ndim), + strides=to_list(layer.strides, ndim), + dilations=to_list(layer.dilation_rate, ndim), + group=getattr(layer, "groups", 1), + auto_pad=auto_pad, + ) + if pads is not None: + conv_attrs["pads"] = pads + + conv_out = f"{prefix}_conv" + nodes.append(oh.make_node("Conv", inputs=conv_inputs, outputs=[conv_out], **conv_attrs)) + current = conv_out + + current = _maybe_quant_output(layer, prefix, current, nodes, initializers, quant_fn) + + if cl: + current = _add_transpose(f"{prefix}_post", current, perm_to_nhwx, nodes) + return current + + +def _add_depthwise_conv(layer, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights): + """PQDepthwiseConv2d. + + Keras kernel: [kH, kW, in, depth_mult] + ONNX Conv with groups=in: weight [in*depth_mult, 1, kH, kW] + """ + cl = _channels_last(layer) + + if cl: + current = _add_transpose(f"{prefix}_pre", current, [0, 3, 1, 2], nodes) + + current = _maybe_quant_input(layer, prefix, current, nodes, initializers, quant_fn) + + kernel_np = _np(layer._kernel) # [kH, kW, in, depth_mult] + in_ch, depth_mult = kernel_np.shape[2], kernel_np.shape[3] + # Rearrange to [in*depth_mult, 1, kH, kW] + kernel_onnx = np.transpose(kernel_np, (2, 3, 0, 1)).reshape(in_ch * depth_mult, 1, *kernel_np.shape[:2]) + + out_channels = kernel_onnx.shape[0] + + if use_qonnx: + k_w, i_w, f_w = layer.weight_quantizer.get_quantization_bits() + wfp_name = f"{prefix}_weight_fp" + initializers.append(onh.from_array(kernel_onnx, name=wfp_name)) + w_nodes, q_weight = _quant_node( + f"{prefix}_weight", + wfp_name, + layer.weight_quantizer.round_mode, + _np(k_w), + _np(i_w), + _np(f_w), + initializers, + overflow_mode=getattr(layer.weight_quantizer, "overflow", "SAT"), + ) + nodes.extend(w_nodes) + elif store_integer_weights: + k_w, i_w, f_w = layer.weight_quantizer.get_quantization_bits() + f_for_onnx = _weight_f_for_onnx(_np(f_w), out_channels) + k_for_onnx = _weight_f_for_onnx(_np(k_w), out_channels) + i_for_onnx = _weight_f_for_onnx(_np(i_w), out_channels) + w_nodes, q_weight = _int_weight_node( + f"{prefix}_weight", kernel_onnx, k_for_onnx, i_for_onnx, f_for_onnx, initializers + ) + nodes.extend(w_nodes) + else: + q_weight = f"{prefix}_weight" + initializers.append(onh.from_array(kernel_onnx, name=q_weight)) + + conv_inputs = [current, q_weight] + + if layer._bias is not None: + bias_np = _np(layer._bias) + if use_qonnx: + k_b, i_b, f_b = layer.bias_quantizer.get_quantization_bits() + bfp_name = f"{prefix}_bias_fp" + initializers.append(onh.from_array(bias_np, name=bfp_name)) + b_nodes, q_bias = _quant_node( + f"{prefix}_bias", + bfp_name, + layer.bias_quantizer.round_mode, + _np(k_b), + _np(i_b), + _np(f_b), + initializers, + overflow_mode=getattr(layer.bias_quantizer, "overflow", "SAT"), + ) + nodes.extend(b_nodes) + elif store_integer_weights: + k_b, i_b, f_b = layer.bias_quantizer.get_quantization_bits() + b_nodes, q_bias = _int_weight_node(f"{prefix}_bias", bias_np, _np(k_b), _np(i_b), _np(f_b), initializers) + nodes.extend(b_nodes) + else: + q_bias = f"{prefix}_bias" + initializers.append(onh.from_array(bias_np, name=q_bias)) + conv_inputs.append(q_bias) + + padding = layer.padding + if isinstance(padding, str): + auto_pad = "SAME_UPPER" if padding == "same" else "VALID" + pads = None + else: + p = list(padding) if hasattr(padding, "__iter__") else [padding, padding] + pads = p + p + auto_pad = "NOTSET" + + to_list = lambda v, n: list(v) if hasattr(v, "__iter__") else [v] * n # noqa: E731 + conv_attrs = dict( + kernel_shape=to_list(layer.kernel_size, 2), + strides=to_list(layer.strides, 2), + dilations=to_list(layer.dilation_rate, 2), + group=in_ch, + auto_pad=auto_pad, + ) + if pads is not None: + conv_attrs["pads"] = pads + + conv_out = f"{prefix}_conv" + nodes.append(oh.make_node("Conv", inputs=conv_inputs, outputs=[conv_out], **conv_attrs)) + current = conv_out + + current = _maybe_quant_output(layer, prefix, current, nodes, initializers, quant_fn) + + if cl: + current = _add_transpose(f"{prefix}_post", current, [0, 2, 3, 1], nodes) + return current + + +def _add_batchnorm(layer, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights): + """PQBatchNormalization / standard BatchNormalization.""" + need_tr, perm_to_nchw, perm_to_nhwx = _bn_transpose_info(layer) + + if need_tr: + current = _add_transpose(f"{prefix}_pre", current, perm_to_nchw, nodes) + + current = _maybe_quant_input(layer, prefix, current, nodes, initializers, quant_fn) + + is_pq = isinstance(layer, PQBatchNormalization) + + gamma_np = _np(layer.gamma) if layer.gamma is not None else None + beta_np = _np(layer.beta) if layer.beta is not None else None + + if gamma_np is None: + # scale=False: use ones + n_ch = _np(layer.moving_mean).shape[0] + gamma_np = np.ones(n_ch, dtype=np.float32) + if beta_np is None: + # center=False: use zeros + n_ch = _np(layer.moving_mean).shape[0] + beta_np = np.zeros(n_ch, dtype=np.float32) + + if is_pq and use_qonnx: + k_w, i_w, f_w = layer.weight_quantizer.get_quantization_bits() + gfp = f"{prefix}_gamma_fp" + initializers.append(onh.from_array(gamma_np, name=gfp)) + g_nodes, q_gamma = _quant_node( + f"{prefix}_gamma", + gfp, + layer.weight_quantizer.round_mode, + _np(k_w), + _np(i_w), + _np(f_w), + initializers, + overflow_mode=getattr(layer.weight_quantizer, "overflow", "SAT"), + ) + nodes.extend(g_nodes) + + k_b, i_b, f_b = layer.bias_quantizer.get_quantization_bits() + bfp = f"{prefix}_beta_fp" + initializers.append(onh.from_array(beta_np, name=bfp)) + b_nodes, q_beta = _quant_node( + f"{prefix}_beta", + bfp, + layer.bias_quantizer.round_mode, + _np(k_b), + _np(i_b), + _np(f_b), + initializers, + overflow_mode=getattr(layer.bias_quantizer, "overflow", "SAT"), + ) + nodes.extend(b_nodes) + elif is_pq and store_integer_weights: + k_w, i_w, f_w = layer.weight_quantizer.get_quantization_bits() + g_nodes, q_gamma = _int_weight_node(f"{prefix}_gamma", gamma_np, _np(k_w), _np(i_w), _np(f_w), initializers) + nodes.extend(g_nodes) + k_b, i_b, f_b = layer.bias_quantizer.get_quantization_bits() + b_nodes, q_beta = _int_weight_node(f"{prefix}_beta", beta_np, _np(k_b), _np(i_b), _np(f_b), initializers) + nodes.extend(b_nodes) + else: + q_gamma = f"{prefix}_gamma" + q_beta = f"{prefix}_beta" + initializers.append(onh.from_array(gamma_np, name=q_gamma)) + initializers.append(onh.from_array(beta_np, name=q_beta)) + + mean_name = f"{prefix}_running_mean" + var_name = f"{prefix}_running_var" + initializers.append(onh.from_array(_np(layer.moving_mean), name=mean_name)) + initializers.append(onh.from_array(_np(layer.moving_variance), name=var_name)) + + bn_out = f"{prefix}_bn" + nodes.append( + oh.make_node( + "BatchNormalization", + inputs=[current, q_gamma, q_beta, mean_name, var_name], + outputs=[bn_out], + epsilon=float(layer.epsilon), + ) + ) + current = bn_out + + if need_tr: + current = _add_transpose(f"{prefix}_post", current, perm_to_nhwx, nodes) + return current + + +def _add_dense_nd(layer, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights): + """PQDense for rank-3 inputs (B, T, E). + + Uses MatMul + Add instead of Gemm so the op works for any rank β‰₯ 2. + The kernel is stored as [out, in] (same layout as _add_dense / _int_weight_node), + then transposed to [in, out] at runtime via a Transpose node so that + MatMul(input, kernel_t) broadcasts correctly over the sequence dimension. + """ + current = _maybe_quant_input(layer, prefix, current, nodes, initializers, quant_fn) + + kernel_np = _np(layer._kernel).T # [out, in] + out_units = kernel_np.shape[0] + + if use_qonnx: + k_w, i_w, f_w = layer.weight_quantizer.get_quantization_bits() + wfp_name = f"{prefix}_weight_fp" + initializers.append(onh.from_array(kernel_np, name=wfp_name)) + w_nodes, q_weight = _quant_node( + f"{prefix}_weight", + wfp_name, + layer.weight_quantizer.round_mode, + _np(k_w), + _np(i_w), + _np(f_w), + initializers, + overflow_mode=getattr(layer.weight_quantizer, "overflow", "SAT"), + ) + nodes.extend(w_nodes) + elif store_integer_weights: + k_w, i_w, f_w = layer.weight_quantizer.get_quantization_bits() + f_for_onnx = _weight_f_for_onnx(_np(f_w), out_units) + k_for_onnx = _weight_f_for_onnx(_np(k_w), out_units) + i_for_onnx = _weight_f_for_onnx(_np(i_w), out_units) + w_nodes, q_weight = _int_weight_node(f"{prefix}_weight", kernel_np, k_for_onnx, i_for_onnx, f_for_onnx, initializers) + nodes.extend(w_nodes) + else: + q_weight = f"{prefix}_weight" + initializers.append(onh.from_array(kernel_np, name=q_weight)) + + # Transpose [out, in] β†’ [in, out] so MatMul(input[..., in], kernel_t[in, out]) works + kernel_t_name = f"{prefix}_weight_t" + nodes.append(oh.make_node("Transpose", inputs=[q_weight], outputs=[kernel_t_name], perm=[1, 0])) + + mm_out = f"{prefix}_mm" + nodes.append(oh.make_node("MatMul", inputs=[current, kernel_t_name], outputs=[mm_out])) + current = mm_out + + if layer._bias is not None: + bias_np = _np(layer._bias) + if use_qonnx: + k_b, i_b, f_b = layer.bias_quantizer.get_quantization_bits() + bfp_name = f"{prefix}_bias_fp" + initializers.append(onh.from_array(bias_np, name=bfp_name)) + b_nodes, q_bias = _quant_node( + f"{prefix}_bias", + bfp_name, + layer.bias_quantizer.round_mode, + _np(k_b), + _np(i_b), + _np(f_b), + initializers, + overflow_mode=getattr(layer.bias_quantizer, "overflow", "SAT"), + ) + nodes.extend(b_nodes) + elif store_integer_weights: + k_b, i_b, f_b = layer.bias_quantizer.get_quantization_bits() + b_nodes, q_bias = _int_weight_node(f"{prefix}_bias", bias_np, _np(k_b), _np(i_b), _np(f_b), initializers) + nodes.extend(b_nodes) + else: + q_bias = f"{prefix}_bias" + initializers.append(onh.from_array(bias_np, name=q_bias)) + add_out = f"{prefix}_bias_add" + nodes.append(oh.make_node("Add", inputs=[current, q_bias], outputs=[add_out])) + current = add_out + + current = _maybe_quant_output(layer, prefix, current, nodes, initializers, quant_fn) + return current + + +def _add_mha(layer, prefix, q_input, k_input, v_input, nodes, initializers, quant_fn, use_qonnx, store_integer_weights): + """Build ONNX nodes for PQMultiheadAttention (Keras version, always batch-first). + + Decomposes multi-head attention into primitive ONNX ops: + + Q/K/V MatMul projections (rank-3 MatMul via _add_dense_nd) + Reshape (B, L, E) β†’ (B, H, L, head_dim) + Transpose + MatMul(Q, K^T) * scale β†’ optional Quant + Softmax β†’ optional Quant + MatMul(attn_weights, V) β†’ optional context Quant + Transpose + Reshape β†’ (B, T, E) + out_proj MatMul + + Returns (out_name, avg_attn_weights_name). + """ + H = layer.num_heads + head_dim = layer.head_dim + E = layer.embed_dim + scale_val = float(layer.scale) + + # --- Q / K / V projections: (B, L, E) β†’ (B, L, E) --- + q_proj_out = _add_dense_nd( + layer.q_proj, f"{prefix}_q_proj", q_input, nodes, initializers, quant_fn, use_qonnx, store_integer_weights + ) + k_proj_out = _add_dense_nd( + layer.k_proj, f"{prefix}_k_proj", k_input, nodes, initializers, quant_fn, use_qonnx, store_integer_weights + ) + v_proj_out = _add_dense_nd( + layer.v_proj, f"{prefix}_v_proj", v_input, nodes, initializers, quant_fn, use_qonnx, store_integer_weights + ) + + # --- Helper: (B, L, E) β†’ (B, H, L, head_dim) using dynamic shapes --- + def _split_heads(x_name, pfx): + shape_out = f"{pfx}_shape" + b_scalar = f"{pfx}_b_sc" + l_scalar = f"{pfx}_l_sc" + b_1d = f"{pfx}_b_1d" + l_1d = f"{pfx}_l_1d" + h_1d_const = f"{pfx}_H_1d" + hd_1d_const = f"{pfx}_hd_1d" + shape_4d = f"{pfx}_shape4d" + reshaped = f"{pfx}_reshaped" + transposed = f"{pfx}_transposed" + idx0 = f"{pfx}_gi0" + idx1 = f"{pfx}_gi1" + ax0 = f"{pfx}_ax0" + + nodes.append(oh.make_node("Shape", inputs=[x_name], outputs=[shape_out])) + initializers.extend( + [ + onh.from_array(np.array(0, dtype=np.int64), name=idx0), + onh.from_array(np.array(1, dtype=np.int64), name=idx1), + onh.from_array(np.array([0], dtype=np.int64), name=ax0), + onh.from_array(np.array([H], dtype=np.int64), name=h_1d_const), + onh.from_array(np.array([head_dim], dtype=np.int64), name=hd_1d_const), + ] + ) + nodes.append(oh.make_node("Gather", inputs=[shape_out, idx0], outputs=[b_scalar])) + nodes.append(oh.make_node("Gather", inputs=[shape_out, idx1], outputs=[l_scalar])) + nodes.append(oh.make_node("Unsqueeze", inputs=[b_scalar, ax0], outputs=[b_1d])) + nodes.append(oh.make_node("Unsqueeze", inputs=[l_scalar, ax0], outputs=[l_1d])) + nodes.append(oh.make_node("Concat", inputs=[b_1d, l_1d, h_1d_const, hd_1d_const], outputs=[shape_4d], axis=0)) + nodes.append(oh.make_node("Reshape", inputs=[x_name, shape_4d], outputs=[reshaped])) + # (B, L, H, head_dim) β†’ (B, H, L, head_dim) + nodes.append(oh.make_node("Transpose", inputs=[reshaped], outputs=[transposed], perm=[0, 2, 1, 3])) + return transposed + + q_h = _split_heads(q_proj_out, f"{prefix}_q") + k_h = _split_heads(k_proj_out, f"{prefix}_k") + v_h = _split_heads(v_proj_out, f"{prefix}_v") + + # --- k^T: (B, H, S, head_dim) β†’ (B, H, head_dim, S) --- + k_t_name = f"{prefix}_k_T" + nodes.append(oh.make_node("Transpose", inputs=[k_h], outputs=[k_t_name], perm=[0, 1, 3, 2])) + + # --- Scaled dot-product scores: (B, H, T, head_dim) @ (B, H, head_dim, S) β†’ (B, H, T, S) --- + raw_scores = f"{prefix}_scores_raw" + scaled_scores = f"{prefix}_scores_scaled" + scale_cst = f"{prefix}_attn_scale" + nodes.append(oh.make_node("MatMul", inputs=[q_h, k_t_name], outputs=[raw_scores])) + initializers.append(onh.from_array(np.array(scale_val, dtype=np.float32), name=scale_cst)) + nodes.append(oh.make_node("Mul", inputs=[raw_scores, scale_cst], outputs=[scaled_scores])) + current = scaled_scores + + # --- Optional attn-score quantization --- + if ( + getattr(layer, "quantize_attn_scores", False) + and hasattr(layer, "attn_score_quantizer") + and getattr(layer, "enable_quantization", True) + ): + q = layer.attn_score_quantizer + k_q, i_q, f_q = q.get_quantization_bits() + q_nodes, current = quant_fn( + f"{prefix}_attn_score_q", + current, + q.round_mode, + _np(k_q), + _np(i_q), + _np(f_q), + initializers, + overflow_mode=getattr(q, "overflow", "SAT"), + ) + nodes.extend(q_nodes) + + # --- Softmax (axis=-1); approximate_softmax falls back to standard Softmax in ONNX --- + attn_w_name = f"{prefix}_attn_weights" + nodes.append(oh.make_node("Softmax", inputs=[current], outputs=[attn_w_name], axis=-1)) + current = attn_w_name + + # --- Optional attn-weight quantization --- + if ( + getattr(layer, "quantize_attn_weights", False) + and hasattr(layer, "attn_weight_quantizer") + and getattr(layer, "enable_quantization", True) + ): + q = layer.attn_weight_quantizer + k_q, i_q, f_q = q.get_quantization_bits() + q_nodes, current = quant_fn( + f"{prefix}_attn_weight_q", + current, + q.round_mode, + _np(k_q), + _np(i_q), + _np(f_q), + initializers, + overflow_mode=getattr(q, "overflow", "SAT"), + ) + nodes.extend(q_nodes) + + # --- Context: (B, H, T, S) @ (B, H, S, head_dim) β†’ (B, H, T, head_dim) --- + ctx_raw = f"{prefix}_ctx_raw" + nodes.append(oh.make_node("MatMul", inputs=[current, v_h], outputs=[ctx_raw])) + current_ctx = ctx_raw + + # --- Optional context quantization --- + if ( + getattr(layer, "quantize_context", False) + and hasattr(layer, "context_quantizer") + and getattr(layer, "enable_quantization", True) + ): + q = layer.context_quantizer + k_q, i_q, f_q = q.get_quantization_bits() + q_nodes, current_ctx = quant_fn( + f"{prefix}_context_q", + current_ctx, + q.round_mode, + _np(k_q), + _np(i_q), + _np(f_q), + initializers, + overflow_mode=getattr(q, "overflow", "SAT"), + ) + nodes.extend(q_nodes) + + # --- Merge heads: (B, H, T, head_dim) β†’ (B, T, E) using dynamic shapes --- + ctx_t = f"{prefix}_ctx_t" + ctx_shape = f"{prefix}_ctx_shape" + ctx_b_sc = f"{prefix}_ctx_b_sc" + ctx_t_sc = f"{prefix}_ctx_t_sc" + ctx_b_1d = f"{prefix}_ctx_b_1d" + ctx_t_1d = f"{prefix}_ctx_t_1d" + ctx_E_1d = f"{prefix}_ctx_E_1d" + ctx_ax0 = f"{prefix}_ctx_ax0" + ctx_gi0 = f"{prefix}_ctx_gi0" + ctx_gi1 = f"{prefix}_ctx_gi1" + ctx_3d = f"{prefix}_ctx_shape3d" + ctx_merged = f"{prefix}_ctx_merged" + + nodes.append(oh.make_node("Transpose", inputs=[current_ctx], outputs=[ctx_t], perm=[0, 2, 1, 3])) + nodes.append(oh.make_node("Shape", inputs=[ctx_t], outputs=[ctx_shape])) + initializers += [ + onh.from_array(np.array(0, dtype=np.int64), name=ctx_gi0), + onh.from_array(np.array(1, dtype=np.int64), name=ctx_gi1), + onh.from_array(np.array([0], dtype=np.int64), name=ctx_ax0), + onh.from_array(np.array([E], dtype=np.int64), name=ctx_E_1d), + ] + nodes.append(oh.make_node("Gather", inputs=[ctx_shape, ctx_gi0], outputs=[ctx_b_sc])) + nodes.append(oh.make_node("Gather", inputs=[ctx_shape, ctx_gi1], outputs=[ctx_t_sc])) + nodes.append(oh.make_node("Unsqueeze", inputs=[ctx_b_sc, ctx_ax0], outputs=[ctx_b_1d])) + nodes.append(oh.make_node("Unsqueeze", inputs=[ctx_t_sc, ctx_ax0], outputs=[ctx_t_1d])) + nodes.append(oh.make_node("Concat", inputs=[ctx_b_1d, ctx_t_1d, ctx_E_1d], outputs=[ctx_3d], axis=0)) + nodes.append(oh.make_node("Reshape", inputs=[ctx_t, ctx_3d], outputs=[ctx_merged])) + + # --- Output projection: (B, T, E) β†’ (B, T, E) --- + out = _add_dense_nd( + layer.out_proj, f"{prefix}_out_proj", ctx_merged, nodes, initializers, quant_fn, use_qonnx, store_integer_weights + ) + + # --- Average attention weights over heads: (B, H, T, S) β†’ (B, T, S) --- + avg_attn = f"{prefix}_avg_attn_weights" + nodes.append(oh.make_node("ReduceMean", inputs=[attn_w_name], outputs=[avg_attn], axes=[1], keepdims=0)) + + return out, avg_attn + + +def _add_avgpool(layer, prefix, current, nodes, initializers, ndim, quant_fn): + cl = _channels_last(layer) + + if cl: + perm_to_nchw = [0, 3, 1, 2] if ndim == 2 else [0, 2, 1] + perm_to_nhwx = [0, 2, 3, 1] if ndim == 2 else [0, 2, 1] + current = _add_transpose(f"{prefix}_pre", current, perm_to_nchw, nodes) + + current = _maybe_quant_input(layer, prefix, current, nodes, initializers, quant_fn) + + to_list = lambda v, n: list(v) if hasattr(v, "__iter__") else [v] * n # noqa: E731 + + pool_out = f"{prefix}_pool" + nodes.append( + oh.make_node( + "AveragePool", + inputs=[current], + outputs=[pool_out], + kernel_shape=to_list(layer.pool_size, ndim), + strides=to_list(layer.strides, ndim), + pads=[0] * (ndim * 2), + count_include_pad=0, + ) + ) + current = pool_out + + current = _maybe_quant_output(layer, prefix, current, nodes, initializers, quant_fn) + + if cl: + current = _add_transpose(f"{prefix}_post", current, perm_to_nhwx, nodes) + return current + + +def _add_global_avgpool(layer, prefix, current, nodes, ndim): + cl = _channels_last(layer) + + if cl: + perm_to_nchw = [0, 3, 1, 2] if ndim == 2 else [0, 2, 1] + current = _add_transpose(f"{prefix}_pre", current, perm_to_nchw, nodes) + + pool_out = f"{prefix}_global_pool" + nodes.append(oh.make_node("GlobalAveragePool", inputs=[current], outputs=[pool_out])) + current = pool_out + + if cl: + # GlobalAveragePool returns [N, C, 1, 1]; emit Flatten to [N, C]. + # Actually after GlobalAveragePool output is [N, C, 1, 1]; transpose back would give + # [N, 1, 1, C] which then needs squeezing β€” that's the same as just squeezing [N, C]. + # Emit Flatten to [N, C] instead of bothering with transpose. + flatten_name = f"{prefix}_flatten" + nodes.append(oh.make_node("Flatten", inputs=[pool_out], outputs=[flatten_name], axis=1)) + current = flatten_name + + return current + + +def _add_pq_activation(layer, prefix, current, nodes, initializers, quant_fn): + """PQActivation: [input QDQ] β†’ [multiplier scale] β†’ activation β†’ [output QDQ]. + + Supported activations: relu, tanh, hard_tanh (= Clip(-1, 1)). + + The optional relu multiplier is baked to a constant: 2^round(m). + """ + # --- optional input quantization --- + current = _maybe_quant_input(layer, prefix, current, nodes, initializers, quant_fn) + + # --- optional learnable multiplier (relu only) --- + if ( + getattr(layer, "use_multiplier", False) + and getattr(layer, "activation_name", "") == "relu" + and hasattr(layer, "multiplier") + ): + m_val = float(np.array(layer.multiplier).ravel()[0]) + scale = float(2.0 ** round(m_val)) + scale_name = f"{prefix}_mul_scale" + scaled_out = f"{prefix}_scaled" + initializers.append(onh.from_array(np.array(scale, dtype=np.float32), name=scale_name)) + nodes.append(oh.make_node("Mul", inputs=[current, scale_name], outputs=[scaled_out])) + current = scaled_out + + # --- activation --- + act = getattr(layer, "activation_name", "relu") + act_out = f"{prefix}_act" + if act == "relu": + nodes.append(oh.make_node("Relu", inputs=[current], outputs=[act_out])) + elif act == "tanh": + nodes.append(oh.make_node("Tanh", inputs=[current], outputs=[act_out])) + elif act == "hard_tanh": + # hard_tanh(x) = clip(x, -1, 1) + cmin_name = f"{prefix}_htanh_min" + cmax_name = f"{prefix}_htanh_max" + initializers += [ + onh.from_array(np.array(-1.0, dtype=np.float32), name=cmin_name), + onh.from_array(np.array(1.0, dtype=np.float32), name=cmax_name), + ] + nodes.append(oh.make_node("Clip", inputs=[current, cmin_name, cmax_name], outputs=[act_out])) + else: + raise TypeError(f"PQActivation: unsupported activation {act!r} for ONNX export") + current = act_out + + # --- optional output quantization --- + current = _maybe_quant_output(layer, prefix, current, nodes, initializers, quant_fn) + return current + + +# --------------------------------------------------------------------------- +# shared layer dispatcher +# --------------------------------------------------------------------------- + + +def _emit_layer( + layer, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights, input_onnx_names=None +): + """Emit ONNX nodes for a single Keras layer. Returns the ONNX output name.""" + + # --- PQuant layers --- + if isinstance(layer, PQMultiheadAttention): + # input_onnx_names = [query, key, value] or [single_input] for self-attention + if len(input_onnx_names) >= 3: + q_in, k_in, v_in = input_onnx_names[0], input_onnx_names[1], input_onnx_names[2] + elif len(input_onnx_names) == 2: + q_in, k_in, v_in = input_onnx_names[0], input_onnx_names[1], input_onnx_names[1] + else: + q_in = k_in = v_in = input_onnx_names[0] + return _add_mha(layer, prefix, q_in, k_in, v_in, nodes, initializers, quant_fn, use_qonnx, store_integer_weights) + + if isinstance(layer, PQActivation): + return _add_pq_activation(layer, prefix, current, nodes, initializers, quant_fn) + + if isinstance(layer, PQDense): + return _add_dense(layer, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights) + + if isinstance(layer, PQDepthwiseConv2d): + return _add_depthwise_conv(layer, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights) + + if isinstance(layer, PQConv2d): + return _add_conv( + layer, + prefix, + current, + nodes, + initializers, + ndim=2, + quant_fn=quant_fn, + use_qonnx=use_qonnx, + store_integer_weights=store_integer_weights, + ) + + if isinstance(layer, PQConv1d): + return _add_conv( + layer, + prefix, + current, + nodes, + initializers, + ndim=1, + quant_fn=quant_fn, + use_qonnx=use_qonnx, + store_integer_weights=store_integer_weights, + ) + + if isinstance(layer, PQBatchNormalization): + return _add_batchnorm(layer, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights) + + # --- Standard Keras layers --- + if isinstance(layer, keras.layers.BatchNormalization): + return _add_batchnorm( + layer, prefix, current, nodes, initializers, quant_fn=quant_fn, use_qonnx=False, store_integer_weights=False + ) + + if isinstance(layer, keras.layers.Conv2D): + # Plain Conv2D (non-PQ): wrap in a minimal shim + _layer = layer + _layer._bias = layer.bias + return _add_conv_plain(layer, prefix, current, nodes, initializers) + + if isinstance(layer, keras.layers.Dense): + out = f"{prefix}_gemm" + w_name = f"{prefix}_weight" + initializers.append(onh.from_array(_np(layer.kernel).T, name=w_name)) # store [out, in] + gemm_inputs = [current, w_name] + if layer.bias is not None: + b_name = f"{prefix}_bias" + initializers.append(onh.from_array(_np(layer.bias), name=b_name)) + gemm_inputs.append(b_name) + nodes.append(oh.make_node("Gemm", inputs=gemm_inputs, outputs=[out], transB=1)) + return out + + if isinstance(layer, (keras.layers.ReLU, keras.layers.Activation)): + activation = ( + layer.activation.__name__ + if isinstance(layer, keras.layers.Activation) and callable(layer.activation) + else getattr(layer, "activation", "relu") + ) + act_name = activation if isinstance(activation, str) else "relu" + out = f"{prefix}_act" + if "relu" in act_name.lower(): + nodes.append(oh.make_node("Relu", inputs=[current], outputs=[out])) + elif "sigmoid" in act_name.lower(): + nodes.append(oh.make_node("Sigmoid", inputs=[current], outputs=[out])) + elif "tanh" in act_name.lower(): + nodes.append(oh.make_node("Tanh", inputs=[current], outputs=[out])) + else: + raise TypeError(f"Unsupported Activation for ONNX export: {act_name!r}") + return out + + if isinstance(layer, keras.layers.Flatten): + out = f"{prefix}_flatten" + nodes.append(oh.make_node("Flatten", inputs=[current], outputs=[out], axis=1)) + return out + + if isinstance(layer, keras.layers.Reshape): + target_shape = list(layer.target_shape) + # Prepend batch dim (-1 means dynamic) + full_shape = [-1] + target_shape + shape_name = f"{prefix}_shape" + out = f"{prefix}_reshape" + initializers.append(onh.from_array(np.array(full_shape, dtype=np.int64), name=shape_name)) + nodes.append(oh.make_node("Reshape", inputs=[current, shape_name], outputs=[out])) + return out + + if isinstance(layer, keras.layers.Add): + assert input_onnx_names is not None and len(input_onnx_names) == 2 + out = f"{prefix}_add" + nodes.append(oh.make_node("Add", inputs=input_onnx_names, outputs=[out])) + return out + + if isinstance(layer, keras.layers.Concatenate): + assert input_onnx_names is not None + axis = layer.axis + # Negative axis: leave as-is; onnx Concat supports negative axes + out = f"{prefix}_concat" + nodes.append(oh.make_node("Concat", inputs=input_onnx_names, outputs=[out], axis=axis)) + return out + + if isinstance(layer, keras.layers.Multiply): + assert input_onnx_names is not None and len(input_onnx_names) == 2 + out = f"{prefix}_mul" + nodes.append(oh.make_node("Mul", inputs=input_onnx_names, outputs=[out])) + return out + + if isinstance(layer, keras.layers.AveragePooling2D): + return _add_avgpool(layer, prefix, current, nodes, initializers, ndim=2, quant_fn=quant_fn) + + if isinstance(layer, keras.layers.AveragePooling1D): + return _add_avgpool(layer, prefix, current, nodes, initializers, ndim=1, quant_fn=quant_fn) + + if isinstance(layer, keras.layers.GlobalAveragePooling2D): + return _add_global_avgpool(layer, prefix, current, nodes, ndim=2) + + if isinstance(layer, keras.layers.GlobalAveragePooling1D): + return _add_global_avgpool(layer, prefix, current, nodes, ndim=1) + + if isinstance(layer, (keras.layers.Dropout,)): + return current # identity at inference + + raise TypeError(f"Unsupported Keras layer type for ONNX export: {type(layer).__name__!r}") + + +def _add_conv_plain(layer, prefix, current, nodes, initializers): + """Emit a plain (non-PQ) Conv2D layer.""" + cl = _channels_last(layer) + if cl: + current = _add_transpose(f"{prefix}_pre", current, [0, 3, 1, 2], nodes) + + kernel_np = _np(layer.kernel) + kernel_onnx = np.transpose(kernel_np, (3, 2, 0, 1)) + w_name = f"{prefix}_weight" + initializers.append(onh.from_array(kernel_onnx, name=w_name)) + conv_inputs = [current, w_name] + + if layer.bias is not None: + b_name = f"{prefix}_bias" + initializers.append(onh.from_array(_np(layer.bias), name=b_name)) + conv_inputs.append(b_name) + + padding = layer.padding + auto_pad = "SAME_UPPER" if padding == "same" else "VALID" + to_list = lambda v, n: list(v) if hasattr(v, "__iter__") else [v] * n # noqa: E731 + conv_attrs = dict( + kernel_shape=to_list(layer.kernel_size, 2), + strides=to_list(layer.strides, 2), + dilations=to_list(layer.dilation_rate, 2), + group=layer.groups, + auto_pad=auto_pad, + ) + conv_out = f"{prefix}_conv" + nodes.append(oh.make_node("Conv", inputs=conv_inputs, outputs=[conv_out], **conv_attrs)) + current = conv_out + + if cl: + current = _add_transpose(f"{prefix}_post", current, [0, 2, 3, 1], nodes) + return current + + +# --------------------------------------------------------------------------- +# Keras functional model graph traversal +# --------------------------------------------------------------------------- + + +def _build_tensor_onnx_map(model): + """ + Return a dict mapping id(KerasTensor) β†’ ONNX tensor name for model.inputs. + Multi-input models are supported; inputs are named "input_0", "input_1", etc. + (or just "input" for single-input models). + """ + tensor_to_onnx = {} + for i, inp in enumerate(model.inputs): + name = "input" if len(model.inputs) == 1 else f"input_{i}" + tensor_to_onnx[id(inp)] = name + return tensor_to_onnx + + +def _inbound_input_names(layer, tensor_to_onnx): + """Return the list of ONNX input names for this layer based on its inbound node.""" + if not layer._inbound_nodes: + return [] + node = layer._inbound_nodes[0] + input_tensors = node.input_tensors + if not isinstance(input_tensors, (list, tuple)): + input_tensors = [input_tensors] + result = [] + for t in input_tensors: + key = id(t) + if key not in tensor_to_onnx: + raise RuntimeError( + f"Layer {layer.name!r}: input tensor not found in tensor_to_onnx map. " + "Ensure model.layers is in topological order." + ) + result.append(tensor_to_onnx[key]) + return result + + +def _register_layer_output(layer, onnx_name, tensor_to_onnx): + """Register the ONNX output name for a layer's output tensor(s). + + onnx_name may be a plain string (single-output layer) or a tuple of strings + (multi-output layer, e.g. PQMultiheadAttention returns (out, avg_attn_weights)). + """ + if not layer._inbound_nodes: + return + node = layer._inbound_nodes[0] + out_tensors = node.output_tensors + if not isinstance(out_tensors, (list, tuple)): + out_tensors = [out_tensors] + if isinstance(onnx_name, (list, tuple)): + for tensor, name in zip(out_tensors, onnx_name): + tensor_to_onnx[id(tensor)] = name + else: + tensor_to_onnx[id(out_tensors[0])] = onnx_name + + +# --------------------------------------------------------------------------- +# main conversion +# --------------------------------------------------------------------------- + + +def convert_to_onnx( + model: keras.Model, + input_shape: tuple, + output_path: str = "model.onnx", + opset: int = 13, + use_qonnx: bool = False, + store_integer_weights: bool = False, + include_clip: bool = True, + batch_size: int | None = None, +) -> onnx.ModelProto: + """ + Convert a Keras functional model of PQuant layers to ONNX or QONNX. + + The model must have apply_final_compression() called on all PQ layers + before calling this function. Only inference-mode semantics are exported. + + Args: + model: Trained keras.Model. Must be a functional model + (built with the Keras functional API or subclassed + models whose layers are accessible via model.layers). + input_shape: Shape of a single sample excluding batch, e.g. (3, 32, 32). + For channels_last Conv models use e.g. (32, 32, 3). + output_path: Where to save the .onnx file. + opset: ONNX opset version (β‰₯13 required for per-channel + DequantizeLinear). + use_qonnx: Emit QONNX Quant custom nodes if True. + store_integer_weights: Store weight initializers as int8/uint8 + + DequantizeLinear instead of float32 (ignored when + use_qonnx=True). + include_clip: Prepend a Clip node before each QuantizeLinear when + True (default). Set to False to emit bare + QuantizeLinear+DequantizeLinear pairs β€” safe when + values are guaranteed in-range at inference time since + QuantizeLinear saturates naturally. Ignored when + use_qonnx=True. + batch_size: If not None, fix the batch dimension of all graph + inputs and outputs to this value. If None (default), + the batch dimension is left dynamic. + + Returns: + The constructed onnx.ModelProto. + """ + quant_fn = _quant_node if use_qonnx else functools.partial(_qdq_node, include_clip=include_clip) + + onnx_nodes: list[onnx.NodeProto] = [] + initializers: list[onnx.TensorProto] = [] + + tensor_to_onnx = _build_tensor_onnx_map(model) + last_output_name: str = "" + + for layer in model.layers: + # Skip InputLayer β€” already seeded in tensor_to_onnx + if isinstance(layer, keras.layers.InputLayer): + continue + + input_onnx_names = _inbound_input_names(layer, tensor_to_onnx) + if not input_onnx_names: + continue + + current = input_onnx_names[0] # primary input (used by single-input layers) + prefix = layer.name.replace("/", "_").replace(":", "_") + + output_name = _emit_layer( + layer, + prefix, + current, + onnx_nodes, + initializers, + quant_fn, + use_qonnx, + store_integer_weights, + input_onnx_names=input_onnx_names, + ) + + _register_layer_output(layer, output_name, tensor_to_onnx) + # For multi-output layers (e.g. MHA returns (out, avg_attn)), track only the + # primary output as the graph's last output name. + last_output_name = output_name[0] if isinstance(output_name, tuple) else output_name + + # Determine output shape via a forward pass + dummy = np.zeros((1, *input_shape), dtype=np.float32) + dummy_out = model(dummy, training=False) + dummy_out_np = np.array(ops.convert_to_numpy(dummy_out)) + batch_dim = batch_size # None β†’ dynamic, int β†’ fixed + output_shape = [batch_dim] + list(dummy_out_np.shape[1:]) + + # Build ONNX graph + if len(model.inputs) == 1: + input_vis = [oh.make_tensor_value_info("input", TensorProto.FLOAT, [batch_dim, *input_shape])] + else: + input_vis = [ + oh.make_tensor_value_info(f"input_{i}", TensorProto.FLOAT, [batch_dim, *input_shape]) + for i in range(len(model.inputs)) + ] + output_vi = oh.make_tensor_value_info(last_output_name, TensorProto.FLOAT, output_shape) + + graph = oh.make_graph( + nodes=onnx_nodes, + name="pquant_keras_onnx", + inputs=input_vis, + outputs=[output_vi], + initializer=initializers, + ) + + opset_imports = [oh.make_opsetid("", opset)] + if use_qonnx: + opset_imports.append(oh.make_opsetid("qonnx.custom_op.general", 1)) + model_proto = oh.make_model(graph, opset_imports=opset_imports) + model_proto.ir_version = 6 + + # ONNX opset >= 9: initializers are implicit constants and must NOT appear in + # graph.input β€” otherwise tools treat weight tensors as runtime inputs. + # Some onnx library versions add them automatically for backward compatibility; + # strip them here so only the actual data inputs remain. + _init_names = {t.name for t in model_proto.graph.initializer} + _data_inputs = [vi for vi in model_proto.graph.input if vi.name not in _init_names] + del model_proto.graph.input[:] + model_proto.graph.input.extend(_data_inputs) + + onnx.checker.check_model(model_proto) + onnx.save(model_proto, output_path) + fmt = "QONNX" if use_qonnx else "ONNX (QDQ)" + logging.info("Saved %s Keras model β†’ %s", fmt, output_path) + return model_proto + + +# --------------------------------------------------------------------------- +# usage example +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + import pquant + from pquant import apply_final_compression + + cfg = pquant.pdp_config() + + inp = keras.Input(shape=(3, 32, 32)) + x = PQConv2d(cfg, filters=16, kernel_size=3, padding="same")(inp) + x = PQBatchNormalization(cfg)(x) + x = keras.layers.ReLU()(x) + x = PQConv2d(cfg, filters=32, kernel_size=3, padding="same")(x) + x = keras.layers.ReLU()(x) + x = keras.layers.Flatten()(x) + x = PQDense(cfg, units=10)(x) + model = keras.Model(inp, x) + + apply_final_compression(model) + + convert_to_onnx(model, input_shape=(3, 32, 32), output_path="model_keras.onnx") + + import onnxruntime as ort + + sess = ort.InferenceSession("model_keras.onnx") + out = sess.run(None, {"input": np.random.randn(2, 3, 32, 32).astype(np.float32)}) + print("Output shape:", out[0].shape) # noqa: T201 diff --git a/src/pquant/core/keras/layers.py b/src/pquant/core/keras/layers.py index 17d5051..d4db6d8 100644 --- a/src/pquant/core/keras/layers.py +++ b/src/pquant/core/keras/layers.py @@ -96,6 +96,7 @@ def __init__( self.use_fitcompress = config.fitcompress_parameters.enable_fitcompress self.hgq_gamma = config.quantization_parameters.hgq_gamma self.granularity = config.quantization_parameters.granularity + self.dynamic_data = config.quantization_parameters.dynamic_data_quantization self.final_compression_done = False self.built = False self.parallelization_factor = -1 @@ -140,6 +141,7 @@ def __init__( is_data=True, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.dynamic_data, ) self.output_quantizer = Quantizer( k=ops.convert_to_tensor(self.k_output), @@ -151,6 +153,7 @@ def __init__( is_data=True, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.dynamic_data, ) def set_enable_pruning(self, enable_pruning): @@ -1307,6 +1310,7 @@ def __init__( self.quantize_input = quantize_input self.quantize_parameters = quantize_parameters self.granularity = config.quantization_parameters.granularity + self.dynamic_data = config.quantization_parameters.dynamic_data_quantization self.config = config self.f_weight = self.f_bias = ops.convert_to_tensor(config.quantization_parameters.default_weight_fractional_bits) self.i_weight = self.i_bias = ops.convert_to_tensor(config.quantization_parameters.default_weight_integer_bits) @@ -1334,6 +1338,7 @@ def build(self, input_shape): is_data=True, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.dynamic_data, ) self.weight_quantizer = Quantizer( k=1.0, @@ -1521,6 +1526,7 @@ def __init__( self.hgq_gamma = config.quantization_parameters.hgq_gamma self.hgq_beta = config.quantization_parameters.hgq_beta self.hgq_heterogeneous = config.quantization_parameters.hgq_heterogeneous + self.dynamic_data = config.quantization_parameters.dynamic_data_quantization self._is_pretraining = True self.quantize_input = quantize_input self.quantize_output = quantize_output @@ -1551,6 +1557,7 @@ def build(self, input_shape): is_data=True, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.dynamic_data, ) self.output_quantizer = Quantizer( k=1.0, @@ -1562,6 +1569,7 @@ def build(self, input_shape): is_data=True, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.dynamic_data, ) self.input_quantizer.build(input_shape) self.output_quantizer.build(self.compute_output_shape(input_shape)) @@ -1703,6 +1711,272 @@ def get_config(self): return super().get_config() +@keras.saving.register_keras_serializable(package="PQuantML") +class PQMultiheadAttention(keras.layers.Layer): + """Multi-head attention with quantization support. + + Uses separate PQDense projections for Q, K, V, and output, and computes + scaled dot-product attention manually. + + Args: + config: PQuant configuration object. + embed_dim: Total embedding dimension. + num_heads: Number of attention heads. + dropout: Dropout probability on attention weights. + bias: Whether to add bias to projection layers. + kdim: Key feature dimension (defaults to embed_dim). + vdim: Value feature dimension (defaults to embed_dim). + quantize_input: Whether to quantize Q/K/V projection inputs. + quantize_output: Whether to quantize projection outputs. + quantize_attn_weights: Whether to quantize attention weights after softmax. + quantize_attn_scores: Whether to quantize attention scores before softmax. + quantize_context: Whether to quantize the context vector before merging heads. + approximate_softmax: Placeholder for approximate softmax (currently uses standard softmax). + in_quant_bits: (k, i, f) bits for input quantization. + weight_quant_bits: (k, i, f) bits for weight quantization. + bias_quant_bits: (k, i, f) bits for bias quantization. + out_quant_bits: (k, i, f) bits for output quantization. + attn_quant_bits: (k, i, f) bits for attention weight quantization. + attn_score_quant_bits: (k, i, f) bits for attention score quantization. + context_quant_bits: (k, i, f) bits for context quantization. + + Call args: + inputs: A tuple (query, key, value) of tensors with shape (batch, seq, features), + or a single tensor for self-attention. + training: Python boolean indicating whether the layer should behave in training mode. + key_padding_mask: Boolean tensor of shape (batch, key_seq). True means the position + should be ignored. + attn_mask: Additive mask of shape (query_seq, key_seq) or + (batch, num_heads, query_seq, key_seq). + need_weights: If True, returns (output, attn_weights). If False, returns (output, None). + """ + + def __init__( + self, + config, + embed_dim: int, + num_heads: int, + dropout: float = 0.0, + bias: bool = True, + kdim: int = None, + vdim: int = None, + quantize_input: bool = True, + quantize_output: bool = False, + quantize_attn_weights: bool = False, + quantize_attn_scores: bool = False, + quantize_context: bool = False, + approximate_softmax: bool = False, + in_quant_bits: Tuple[T, T, T] = None, + weight_quant_bits: Tuple[T, T, T] = None, + bias_quant_bits: Tuple[T, T, T] = None, + out_quant_bits: Tuple[T, T, T] = None, + attn_quant_bits: Tuple[T, T, T] = None, + attn_score_quant_bits: Tuple[T, T, T] = None, + context_quant_bits: Tuple[T, T, T] = None, + **kwargs, + ): + super().__init__(**kwargs) + assert embed_dim % num_heads == 0, "embed_dim must be divisible by num_heads" + + if isinstance(config, dict): + config = PQConfig.load_from_config(config) + + self.config = config + self.embed_dim = embed_dim + self.num_heads = num_heads + self.head_dim = embed_dim // num_heads + self.dropout_rate = dropout + self.use_bias = bias + self.kdim = kdim if kdim is not None else embed_dim + self.vdim = vdim if vdim is not None else embed_dim + self.quantize_attn_weights = quantize_attn_weights + self.quantize_attn_scores = quantize_attn_scores + self.quantize_context = quantize_context + self.approximate_softmax = approximate_softmax + self.scale = self.head_dim**-0.5 + self.enable_quantization = config.quantization_parameters.enable_quantization + self.use_hgq = config.quantization_parameters.use_high_granularity_quantization + + self.in_quant_bits = in_quant_bits + self.weight_quant_bits = weight_quant_bits + self.bias_quant_bits = bias_quant_bits + self.out_quant_bits = out_quant_bits + self.attn_quant_bits = attn_quant_bits + self.attn_score_quant_bits = attn_score_quant_bits + self.context_quant_bits = context_quant_bits + + proj_kwargs = dict( + use_bias=bias, + quantize_input=quantize_input, + quantize_output=quantize_output, + in_quant_bits=in_quant_bits, + weight_quant_bits=weight_quant_bits, + bias_quant_bits=bias_quant_bits, + out_quant_bits=out_quant_bits, + ) + self.q_proj = PQDense(config, embed_dim, enable_pruning=False, **proj_kwargs) + self.k_proj = PQDense(config, embed_dim, enable_pruning=False, **proj_kwargs) + self.v_proj = PQDense(config, embed_dim, enable_pruning=False, **proj_kwargs) + self.out_proj = PQDense(config, embed_dim, **proj_kwargs) + + self.attn_dropout = keras.layers.Dropout(dropout) if dropout > 0.0 else None + + def _make_data_quantizer(bits): + if bits is not None: + k, i, f = bits + else: + k = config.quantization_parameters.default_data_keep_negatives + i = config.quantization_parameters.default_data_integer_bits + f = config.quantization_parameters.default_data_fractional_bits + return Quantizer( + k=ops.convert_to_tensor(k), + i=ops.convert_to_tensor(i), + f=ops.convert_to_tensor(f), + overflow=config.quantization_parameters.overflow_mode_data, + round_mode=config.quantization_parameters.round_mode, + is_heterogeneous=config.quantization_parameters.use_high_granularity_quantization, + is_data=True, + hgq_gamma=config.quantization_parameters.hgq_gamma, + place="datalane", + dynamic_data=config.quantization_parameters.dynamic_data_quantization, + ) + + if quantize_attn_weights: + self.attn_weight_quantizer = _make_data_quantizer(attn_quant_bits) + if quantize_attn_scores: + self.attn_score_quantizer = _make_data_quantizer(attn_score_quant_bits) + if quantize_context: + self.context_quantizer = _make_data_quantizer(context_quant_bits) + + def call( + self, + inputs, + training=None, + key_padding_mask=None, + attn_mask=None, + need_weights=True, + ): + if isinstance(inputs, (list, tuple)): + if len(inputs) == 3: + query, key, value = inputs + elif len(inputs) == 2: + query, key = inputs + value = key + else: + query = key = value = inputs[0] + else: + query = key = value = inputs + + batch_size = ops.shape(query)[0] + query_len = ops.shape(query)[1] + key_len = ops.shape(key)[1] + + q = self.q_proj(query, training=training) # (B, T, E) + k = self.k_proj(key, training=training) # (B, S, E) + v = self.v_proj(value, training=training) # (B, S, E) + + # Reshape to (B, H, T/S, head_dim) + q = ops.reshape(q, (batch_size, query_len, self.num_heads, self.head_dim)) + q = ops.transpose(q, (0, 2, 1, 3)) + k = ops.reshape(k, (batch_size, key_len, self.num_heads, self.head_dim)) + k = ops.transpose(k, (0, 2, 1, 3)) + v = ops.reshape(v, (batch_size, key_len, self.num_heads, self.head_dim)) + v = ops.transpose(v, (0, 2, 1, 3)) + + # Scaled dot-product attention scores: (B, H, T, S) + attn_scores = ops.matmul(q, ops.transpose(k, (0, 1, 3, 2))) * self.scale + + if attn_mask is not None: + if ops.ndim(attn_mask) == 2: + # (T, S) -> (1, 1, T, S) + attn_mask = ops.reshape(attn_mask, (1, 1, query_len, key_len)) + elif ops.ndim(attn_mask) == 3: + # (B*H, T, S) -> (B, H, T, S) + attn_mask = ops.reshape(attn_mask, (batch_size, self.num_heads, query_len, key_len)) + attn_scores = attn_scores + ops.cast(attn_mask, attn_scores.dtype) + + if key_padding_mask is not None: + # key_padding_mask: (B, S), True means ignore -> (B, 1, 1, S) + mask = ops.cast(key_padding_mask, attn_scores.dtype) + mask = ops.reshape(mask, (batch_size, 1, 1, key_len)) + attn_scores = attn_scores + mask * -1e9 + + if self.quantize_attn_scores and self.enable_quantization: + attn_scores = self.attn_score_quantizer(attn_scores, training=training) + + attn_weights = ops.softmax(attn_scores, axis=-1) + + if self.quantize_attn_weights and self.enable_quantization: + attn_weights = self.attn_weight_quantizer(attn_weights, training=training) + + if self.attn_dropout is not None: + attn_weights = self.attn_dropout(attn_weights, training=training) + + # Weighted sum of values: (B, H, T, head_dim) + out = ops.matmul(attn_weights, v) + + if self.quantize_context and self.enable_quantization: + out = self.context_quantizer(out, training=training) + + # Merge heads: (B, T, E) + out = ops.transpose(out, (0, 2, 1, 3)) + out = ops.reshape(out, (batch_size, query_len, self.embed_dim)) + out = self.out_proj(out, training=training) + + if self.use_hgq: + if self.quantize_attn_scores: + self.add_loss(self.attn_score_quantizer.hgq_loss()) + if self.quantize_attn_weights: + self.add_loss(self.attn_weight_quantizer.hgq_loss()) + if self.quantize_context: + self.add_loss(self.context_quantizer.hgq_loss()) + + if need_weights: + # Average attention weights over heads: (B, T, S) + return out, ops.mean(attn_weights, axis=1) + return out, None + + def get_config(self): + config = super().get_config() + config.update( + { + "config": self.config.get_dict(), + "embed_dim": self.embed_dim, + "num_heads": self.num_heads, + "dropout": self.dropout_rate, + "bias": self.use_bias, + "kdim": self.kdim, + "vdim": self.vdim, + "quantize_input": self.q_proj.quantize_input, + "quantize_output": self.q_proj.quantize_output, + "quantize_attn_weights": self.quantize_attn_weights, + "quantize_attn_scores": self.quantize_attn_scores, + "quantize_context": self.quantize_context, + "approximate_softmax": self.approximate_softmax, + "in_quant_bits": self.in_quant_bits, + "weight_quant_bits": self.weight_quant_bits, + "bias_quant_bits": self.bias_quant_bits, + "out_quant_bits": self.out_quant_bits, + "attn_quant_bits": self.attn_quant_bits, + "attn_score_quant_bits": self.attn_score_quant_bits, + "context_quant_bits": self.context_quant_bits, + } + ) + return config + + @classmethod + def from_config(cls, config): + config = config.copy() + config.pop("q_proj", None) + config.pop("k_proj", None) + config.pop("v_proj", None) + config.pop("out_proj", None) + config.pop("attn_weight_quantizer", None) + config.pop("attn_score_quantizer", None) + config.pop("context_quantizer", None) + return cls(**config) + + def call_post_round_functions(model, rewind, rounds, r): last_round = r == rounds - 1 if rewind == "every-round": @@ -1721,6 +1995,9 @@ def apply_final_compression(model): layer.input_quantizer.apply_final_compression() if hasattr(layer, "output_quantizer"): layer.output_quantizer.apply_final_compression() + elif isinstance(layer, PQMultiheadAttention): + for proj in (layer.q_proj, layer.k_proj, layer.v_proj, layer.out_proj): + proj.apply_final_compression() return model diff --git a/src/pquant/core/keras/quantizer.py b/src/pquant/core/keras/quantizer.py index 8cc63af..aa11d54 100644 --- a/src/pquant/core/keras/quantizer.py +++ b/src/pquant/core/keras/quantizer.py @@ -22,6 +22,7 @@ def __init__( granularity="per_tensor", hgq_gamma=0, place="datalane", + dynamic_data=True, ): super().__init__() self.k_init = float(k) @@ -32,18 +33,33 @@ def __init__( self.round_mode = round_mode self.use_hgq = is_heterogeneous self.is_data = is_data + self.dynamic_data = dynamic_data self.place = place + self.granularity = granularity.value if isinstance(granularity, Enum) else granularity self.quantizer = create_quantizer( self.k_init, self.i_init, self.f_init, self.overflow, self.round_mode, self.use_hgq, self.is_data, place ) self.is_pretraining = True self.hgq_gamma = hgq_gamma - if isinstance(granularity, Enum): - self.granularity = granularity.value - else: - self.granularity = granularity - def compute_dynamic_bits(self, x): + def calculate_bits_from_abs(self, abs_x): + m = ops.ceil(ops.log(abs_x + 1e-6) / ops.log(2.0)) + int_bits = ops.maximum(m, 0.0) + b = self.b if hasattr(self, "b") else self.b_init + frac_bits = ops.maximum(b - int_bits - self.k_init, 0.0) + return int_bits, frac_bits + + def compute_data_dynamic_bits(self, x): + if not self.dynamic_data: + _, i, f = self.get_quantization_bits() + return i, f + abs_x = ops.max(ops.abs(x)) + return self.calculate_bits_from_abs(abs_x) + + def compute_weight_dynamic_bits(self, x): + if self.granularity == "per_tensor": + _, i, f = self.get_quantization_bits() + return i, f if self.granularity == "per_channel": if ops.ndim(x) == 2: abs_x = ops.max(ops.abs(x), axis=0, keepdims=True) @@ -57,11 +73,12 @@ def compute_dynamic_bits(self, x): abs_x = ops.abs(x) else: raise ValueError(f"compute_dynamic_bits called for granularity={self.granularity}") - m = ops.ceil(ops.log(abs_x + 1e-6) / ops.log(2.0)) - int_bits = ops.maximum(m, 0.0) - b = self.b if hasattr(self, "b") else self.b_init - frac_bits = ops.maximum(b - int_bits - self.k_init, 0.0) - return int_bits, frac_bits + return self.calculate_bits_from_abs(abs_x) + + def compute_dynamic_bits(self, x): + if self.is_data: + return self.compute_data_dynamic_bits(x) + return self.compute_weight_dynamic_bits(x) def build(self, input_shape): if self.use_hgq: @@ -133,8 +150,6 @@ def call(self, x, training=None): return self.quantizer(x, training=training) if not training: return self.quantizer(x, k=self.k, i=self.i, f=self.f, training=training) - elif self.granularity == "per_tensor": - i, f = self.i, self.f else: i, f = self.compute_dynamic_bits(x) self.i.assign(i) @@ -159,6 +174,7 @@ def from_config(cls, config): is_data=config.pop("is_data"), granularity=config.pop("granularity"), place=config.pop("place"), + dynamic_data=config.pop("dynamic_data", True), ) if use_hgq: @@ -180,6 +196,7 @@ def get_config(self): "is_heterogeneous": self.use_hgq, "granularity": self.granularity, "place": self.place, + "dynamic_data": self.dynamic_data, } ) if self.use_hgq: diff --git a/src/pquant/core/torch/activations.py b/src/pquant/core/torch/activations.py index 4630a04..63e9b5e 100644 --- a/src/pquant/core/torch/activations.py +++ b/src/pquant/core/torch/activations.py @@ -22,7 +22,13 @@ def hard_tanh(x): return 2.0 * hard_sigmoid(x) - 1.0 -activation_registry = {"relu": relu, "tanh": tanh, "hard_tanh": hard_tanh} +activation_registry = { + "relu": relu, + "tanh": tanh, + "hard_tanh": hard_tanh, + "leaky_relu": nn.LeakyReLU(negative_slope=0.1015625), + "gelu": nn.GELU(), +} class PQActivation(nn.Module): @@ -69,7 +75,7 @@ def __init__( self.hgq_gamma = config.quantization_parameters.hgq_gamma self.hgq_heterogeneous = config.quantization_parameters.hgq_heterogeneous self.use_fitcompress = config.fitcompress_parameters.enable_fitcompress - self.granularity = config.quantization_parameters.granularity + self.dynamic_data = config.quantization_parameters.dynamic_data_quantization self.post_fitcompress_calibration = False self.saved_inputs = [] @@ -92,6 +98,7 @@ def check_is_built(self, input_shape): is_heterogeneous=self.use_hgq, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.dynamic_data, ) self.input_quantizer = Quantizer( k=self.k_input, @@ -103,6 +110,7 @@ def check_is_built(self, input_shape): is_heterogeneous=self.use_hgq, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.dynamic_data, ) if self.use_hgq: self.input_quantizer.quantizer.build(input_shape) diff --git a/src/pquant/core/torch/convert_to_onnx.py b/src/pquant/core/torch/convert_to_onnx.py new file mode 100644 index 0000000..01825c4 --- /dev/null +++ b/src/pquant/core/torch/convert_to_onnx.py @@ -0,0 +1,1865 @@ +""" +Convert a PQuant model to ONNX or QONNX format. + +Pass ``use_qonnx=True`` to emit QONNX ``Quant`` custom nodes (requires the +qonnx runtime). Pass ``use_qonnx=False`` (default) to emit standard +``Clip + QuantizeLinear + DequantizeLinear`` nodes runnable with plain +onnxruntime. + +Fixed-point (k, i, f) mapping +------------------------------ +QONNX: + scale = 2^(-f) + zero_point = 0 + bit_width = k + i + f + signed = int(k) + +Standard ONNX (QDQ): + scale = 2^(-f) + zero_point = 0 (int8 signed, uint8 unsigned) + clip range = [-2^i, 2^i - 2^(-f)] signed + = [0, 2^i - 2^(-f)] unsigned + Rounding is always nearest-even (QuantizeLinear behaviour). + Weights are stored as plain float32 initializers β€” after + apply_final_compression() they are already on the fixed-point grid. +""" + +import functools +import logging +import operator as _operator +import os + +import numpy as np +import onnx +import onnx.helper as oh +import onnx.numpy_helper as onh +import torch +import torch.fx as _fx +import torch.nn as nn +import torch.nn.functional as _F +from onnx import TensorProto + +os.environ["KERAS_BACKEND"] = "torch" # must be set before any keras/pquant import + +from pquant.core.torch.activations import PQActivation # noqa: E402 +from pquant.core.torch.layers import ( # noqa: E402 + PQAvgPool1d, + PQAvgPool2d, + PQBatchNorm1d, + PQBatchNorm2d, + PQConv1d, + PQConv2d, + PQDense, + PQLayerNorm, + PQMultiheadAttention, +) + +# --------------------------------------------------------------------------- +# QONNX Quant node +# --------------------------------------------------------------------------- + +ROUND_MODE_MAP = { + "TRN": "FLOOR", + "RND": "ROUND", + "RND_CONV": "ROUND", + "TRN_ZERO": "TRUNCATE", + "RND_ZERO": "ROUND", + "RND_MIN_INF": "FLOOR", + "RND_INF": "ROUND", +} + + +def _quant_node(name_prefix, input_name, rounding_mode, k, i, f, initializers, overflow_mode="SAT"): + """Build a QONNX Quant node. Returns ([node], output_name). + + QONNX Quant is per-tensor only. If i/f are per-channel or per-weight tensors + (non-scalar), collapse to the broadest range: min(f) / max(i) ensures no channel + overflows at the cost of slightly coarser quantization for small-value channels. + """ + k_val = int(k.item()) + if hasattr(f, "numel") and f.numel() > 1: + i = i.reshape(-1).max() + f = f.reshape(-1).min() + i_val = float(i.item()) + f_val = float(f.item()) + scale = float(2.0 ** (-f_val)) + bit_width = float(k_val + i_val + f_val) + qonnx_rnd = ROUND_MODE_MAP.get(rounding_mode, "ROUND") + narrow = 1 if (k_val == 1 and overflow_mode == "SAT_SYM") else 0 + + scale_name = f"{name_prefix}_scale" + zp_name = f"{name_prefix}_zero_point" + bw_name = f"{name_prefix}_bit_width" + out_name = f"{name_prefix}_quantized" + + initializers.append(onh.from_array(np.array(scale, dtype=np.float32), name=scale_name)) + initializers.append(onh.from_array(np.array(0.0, dtype=np.float32), name=zp_name)) + initializers.append(onh.from_array(np.array(bit_width, dtype=np.float32), name=bw_name)) + + node = oh.make_node( + op_type="Quant", + inputs=[input_name, scale_name, zp_name, bw_name], + outputs=[out_name], + domain="qonnx.custom_op.general", + signed=k_val, + narrow=narrow, + rounding_mode=qonnx_rnd, + ) + return [node], out_name + + +# --------------------------------------------------------------------------- +# Standard ONNX QDQ triple +# --------------------------------------------------------------------------- + + +def _qdq_node( + name_prefix, input_name, rounding_mode, k, i, f, initializers, overflow_mode="SAT", include_clip=True +): # noqa: ARG001 + """Build QuantizeLinear+DequantizeLinear nodes, optionally preceded by a Clip. + + Returns ([nodes], output_name). Set include_clip=False to skip the Clip node + (safe when values are guaranteed in-range at inference time, since + QuantizeLinear saturates naturally). + """ + k_val = int(k.item()) + i_val = float(i.item()) + f_val = float(f.item()) + scale = float(2.0 ** (-f_val)) + signed = k_val == 1 + + clip_max = float(2.0**i_val - 2.0 ** (-f_val)) + if not signed: + clip_min = 0.0 + elif overflow_mode == "SAT_SYM": + clip_min = -clip_max + else: + clip_min = float(-(2.0**i_val)) + zp_val = np.int8(0) if signed else np.uint8(0) + + scale_name = f"{name_prefix}_scale" + zp_name = f"{name_prefix}_zero_point" + quantized_name = f"{name_prefix}_quantized" + out_name = f"{name_prefix}_dequantized" + + initializers += [ + onh.from_array(np.array(scale, dtype=np.float32), name=scale_name), + onh.from_array(np.array(zp_val), name=zp_name), + ] + + if include_clip: + clip_min_name = f"{name_prefix}_clip_min" + clip_max_name = f"{name_prefix}_clip_max" + clipped_name = f"{name_prefix}_clipped" + initializers += [ + onh.from_array(np.array(clip_min, dtype=np.float32), name=clip_min_name), + onh.from_array(np.array(clip_max, dtype=np.float32), name=clip_max_name), + ] + nodes = [ + oh.make_node("Clip", inputs=[input_name, clip_min_name, clip_max_name], outputs=[clipped_name]), + oh.make_node("QuantizeLinear", inputs=[clipped_name, scale_name, zp_name], outputs=[quantized_name]), + ] + else: + nodes = [ + oh.make_node("QuantizeLinear", inputs=[input_name, scale_name, zp_name], outputs=[quantized_name]), + ] + nodes.append(oh.make_node("DequantizeLinear", inputs=[quantized_name, scale_name, zp_name], outputs=[out_name])) + return nodes, out_name + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + + +def _int_weight_node(name_prefix, weight_np, k, i, f, initializers): # noqa: ARG001 (i unused) + """ + Store a weight tensor as int8/uint8 + DequantizeLinear. + + weight_np must already be on the fixed-point grid (guaranteed after + apply_final_compression). Converts by dividing by the scale and casting β€” + no re-rounding needed. + + Granularity handling: + - per-tensor (f is scalar): single scale, standard DequantizeLinear. + - per-channel (f has shape [out, 1, ...]): 1D scale with axis=0. + All weights in a channel share the same f so the conversion is exact. + - per-weight (f is fully per-element): ONNX has no per-weight quantization; + falls back to float32 storage (no DequantizeLinear node). + + Returns ([node], output_name). + """ + k_val = int(k.item()) if hasattr(k, "item") else int(k) + dtype = np.int8 if k_val == 1 else np.uint8 + out_channels = weight_np.shape[0] + out_name = f"{name_prefix}_dequantized" + + f_t = f.detach().cpu() if hasattr(f, "detach") else torch.as_tensor(f) + + if f_t.numel() == 1: + # per-tensor + scale_np = np.array(float(2.0 ** (-f_t.item())), dtype=np.float32) + int_weights = np.round(weight_np / float(scale_np)).astype(dtype) + per_channel = False + else: + f_np = f_t.float().numpy().reshape(out_channels, -1) + if np.allclose(f_np, f_np[:, :1]): + # per-channel: all elements within an output channel share one f + f_1d = f_np[:, 0] + scale_np = (2.0 ** (-f_1d)).astype(np.float32) + bcast = scale_np.reshape((out_channels,) + (1,) * (weight_np.ndim - 1)) + int_weights = np.round(weight_np / bcast).astype(dtype) + per_channel = True + else: + # per-weight: ONNX cannot represent this; store as float32 + float_name = f"{name_prefix}_float" + initializers.append(onh.from_array(weight_np, name=float_name)) + return [], float_name + + int_name = f"{name_prefix}_int" + scale_name = f"{name_prefix}_dq_scale" + zp_name = f"{name_prefix}_dq_zp" + + zp_np = np.zeros(out_channels if per_channel else 1, dtype=dtype) + initializers += [ + onh.from_array(int_weights, name=int_name), + onh.from_array(scale_np, name=scale_name), + onh.from_array(zp_np if per_channel else np.array(dtype(0)), name=zp_name), + ] + node_kwargs = {"axis": 0} if per_channel else {} + node = oh.make_node("DequantizeLinear", inputs=[int_name, scale_name, zp_name], outputs=[out_name], **node_kwargs) + return [node], out_name + + +def _torch_padding_to_onnx(padding, ndim): + if isinstance(padding, int): + padding = (padding,) * ndim + return list(padding) + list(padding) + + +def _maybe_quant_input(module, prefix, current, nodes, initializers, quant_fn): + if ( + getattr(module, "input_quantizer", None) is not None + and getattr(module, "quantize_input", True) + and getattr(module, "enable_quantization", True) + ): + q = module.input_quantizer + k, i, f = q.get_quantization_bits() + new_nodes, current = quant_fn( + f"{prefix}_in", current, q.round_mode, k, i, f, initializers, overflow_mode=getattr(q, "overflow", "SAT") + ) + nodes.extend(new_nodes) + return current + + +def _maybe_quant_output(module, prefix, current, nodes, initializers, quant_fn): + if ( + getattr(module, "output_quantizer", None) is not None + and getattr(module, "quantize_output", False) + and getattr(module, "enable_quantization", True) + ): + q = module.output_quantizer + k, i, f = q.get_quantization_bits() + new_nodes, current = quant_fn( + f"{prefix}_out", current, q.round_mode, k, i, f, initializers, overflow_mode=getattr(q, "overflow", "SAT") + ) + nodes.extend(new_nodes) + return current + + +# --------------------------------------------------------------------------- +# per-layer graph builders +# --------------------------------------------------------------------------- + + +def _add_dense_integer(module, prefix, current, nodes, initializers): + """Dense layer using MatMulInteger for true integer arithmetic. + + Flow: + float β†’ Clip+QuantizeLinear β†’ int8 ─┐ + β”œβ”€ MatMulInteger β†’ int32 + int8 weights (pre-transposed) β”€β”€β”€β”€β”€β”€β”€β”˜ + β†’ Add int32 bias + β†’ DequantizeLinear(scale = s_x * s_w) β†’ float + + The inner product accumulates in int32; there is no float Gemm. + A single DequantizeLinear at the end converts back to float for activations. + Per-channel weights use axis=1 on the output DequantizeLinear. + """ + if not (getattr(module, "input_quantizer", None) and getattr(module, "quantize_input", True)): + raise ValueError(f"{prefix}: integer_ops requires quantize_input=True on the layer") + + # --- Input: Clip + QuantizeLinear β†’ int8 (stop before DequantizeLinear) --- + k_x, i_x, f_x = module.input_quantizer.get_quantization_bits() + k_x_val = int(k_x.item()) + i_x_val = float(i_x.item()) + f_x_val = float(f_x.item()) + s_x = float(2.0 ** (-f_x_val)) + signed_x = k_x_val == 1 + + clip_min_x = float(-(2.0**i_x_val)) if signed_x else 0.0 + clip_max_x = float(2.0**i_x_val - 2.0 ** (-f_x_val)) + zp_x_np = np.int8(0) if signed_x else np.uint8(0) + + clip_min_name = f"{prefix}_in_clip_min" + clip_max_name = f"{prefix}_in_clip_max" + scale_x_name = f"{prefix}_in_scale" + zp_x_name = f"{prefix}_in_zp" + x_int_name = f"{prefix}_in_int" + + initializers += [ + onh.from_array(np.array(clip_min_x, dtype=np.float32), name=clip_min_name), + onh.from_array(np.array(clip_max_x, dtype=np.float32), name=clip_max_name), + onh.from_array(np.array(s_x, dtype=np.float32), name=scale_x_name), + onh.from_array(np.array(zp_x_np), name=zp_x_name), + ] + nodes += [ + oh.make_node("Clip", inputs=[current, clip_min_name, clip_max_name], outputs=[f"{prefix}_in_clipped"]), + oh.make_node("QuantizeLinear", inputs=[f"{prefix}_in_clipped", scale_x_name, zp_x_name], outputs=[x_int_name]), + ] + + # --- Weights: stored pre-transposed as int8 so MatMulInteger needs no Transpose --- + # PyTorch weight shape: [out, in]. MatMulInteger(A, B) = A @ B, so we need [in, out]. + weight_np = module._weight.detach().cpu().numpy().astype(np.float32) + k_w, _, f_w = module.weight_quantizer.get_quantization_bits() + k_w_val = int(k_w.item()) if hasattr(k_w, "item") else int(k_w) + dtype_w = np.int8 if k_w_val == 1 else np.uint8 + out_ch = weight_np.shape[0] + + f_w_t = f_w.detach().cpu() if hasattr(f_w, "detach") else torch.as_tensor(f_w) + if f_w_t.numel() == 1: + f_w_1d = np.array([float(f_w_t.item())]) + per_channel_w = False + else: + f_w_2d = f_w_t.float().numpy().reshape(out_ch, -1) + f_w_1d = f_w_2d.min(axis=1) # min f β†’ max scale β†’ covers all values + per_channel_w = True + + s_w_1d = (2.0 ** (-f_w_1d)).astype(np.float32) # shape [1] or [out] + bcast_s_w = s_w_1d.reshape((out_ch,) + (1,) * (weight_np.ndim - 1)) if per_channel_w else float(s_w_1d[0]) + # Transpose before storing so MatMulInteger can use it without a runtime Transpose node + int_weights_T = np.round(weight_np / bcast_s_w).astype(dtype_w).T # [in, out] + + zp_w_np = np.array(dtype_w(0)) # scalar zero-point; zero for symmetric quantization + w_int_name = f"{prefix}_weight_int" + w_zp_name = f"{prefix}_weight_zp" + initializers += [ + onh.from_array(int_weights_T, name=w_int_name), + onh.from_array(zp_w_np, name=w_zp_name), + ] + + # --- MatMulInteger([batch, in], [in, out]) β†’ int32 [batch, out] --- + y_int_name = f"{prefix}_matmul_int" + nodes.append( + oh.make_node( + "MatMulInteger", + inputs=[x_int_name, w_int_name, zp_x_name, w_zp_name], + outputs=[y_int_name], + ) + ) + + # --- Bias added in int32 domain: bias_int[c] = round(bias[c] / (s_x * s_w[c])) --- + current_int32 = y_int_name + if module._bias is not None: + bias_np = module._bias.detach().cpu().numpy().astype(np.float32) + combined_s = s_x * s_w_1d # shape [1] or [out] + bias_int32 = np.round(bias_np / (combined_s if per_channel_w else float(combined_s[0]))).astype(np.int32) + bias_int_name = f"{prefix}_bias_int" + y_biased_name = f"{prefix}_matmul_biased" + initializers.append(onh.from_array(bias_int32, name=bias_int_name)) + nodes.append(oh.make_node("Add", inputs=[current_int32, bias_int_name], outputs=[y_biased_name])) + current_int32 = y_biased_name + + # --- DequantizeLinear: int32 β†’ float32 using combined scale s_x * s_w --- + # Per-channel: axis=1 because the output tensor is [batch, out] and out is axis 1. + combined_scale_name = f"{prefix}_combined_scale" + combined_zp_name = f"{prefix}_combined_zp" + + if per_channel_w: + combined_scale_np = (s_x * s_w_1d).astype(np.float32) # [out] + combined_zp_np = np.zeros(out_ch, dtype=np.int32) + dql_kwargs = {"axis": 1} + else: + combined_scale_np = np.array(float(s_x * s_w_1d[0]), dtype=np.float32) + combined_zp_np = np.array(np.int32(0)) + dql_kwargs = {} + + initializers += [ + onh.from_array(combined_scale_np, name=combined_scale_name), + onh.from_array(combined_zp_np, name=combined_zp_name), + ] + y_float_name = f"{prefix}_dequantized" + nodes.append( + oh.make_node( + "DequantizeLinear", + inputs=[current_int32, combined_scale_name, combined_zp_name], + outputs=[y_float_name], + **dql_kwargs, + ) + ) + current = y_float_name + + # Optional output quantization (e.g. last layer with quantize_output=True) + current = _maybe_quant_output(module, prefix, current, nodes, initializers, _qdq_node) + return current + + +def _add_dense_nd(module, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights): + """Dense (linear) projection via MatMul, supporting input of any rank β‰₯ 2. + + Identical logic to _add_dense but emits ``MatMul(input, W_T)`` instead of + ``Gemm(input, W, transB=1)`` so it accepts (B, T, E) inputs (e.g. from MHA + projections) as well as the usual 2-D (batch, features) inputs. + Weight is stored pre-transposed as [in, out] to avoid a runtime Transpose node. + """ + current = _maybe_quant_input(module, prefix, current, nodes, initializers, quant_fn) + + weight_np = module._weight.detach().cpu().numpy().astype(np.float32) # [out, in] + if use_qonnx: + weight_fp_name = f"{prefix}_weight_fp" + initializers.append(onh.from_array(weight_np, name=weight_fp_name)) + k_w, i_w, f_w = module.weight_quantizer.get_quantization_bits() + w_nodes, q_weight_raw = _quant_node( + f"{prefix}_weight", + weight_fp_name, + module.weight_quantizer.round_mode, + k_w, + i_w, + f_w, + initializers, + overflow_mode=getattr(module.weight_quantizer, "overflow", "SAT"), + ) + nodes.extend(w_nodes) + q_weight_t = f"{prefix}_weight_T" + nodes.append(oh.make_node("Transpose", inputs=[q_weight_raw], outputs=[q_weight_t], perm=[1, 0])) + q_weight = q_weight_t + elif store_integer_weights: + k_w, i_w, f_w = module.weight_quantizer.get_quantization_bits() + w_nodes, q_weight_stored = _int_weight_node(f"{prefix}_weight", weight_np, k_w, i_w, f_w, initializers) + nodes.extend(w_nodes) + q_weight_t = f"{prefix}_weight_T" + nodes.append(oh.make_node("Transpose", inputs=[q_weight_stored], outputs=[q_weight_t], perm=[1, 0])) + q_weight = q_weight_t + else: + q_weight = f"{prefix}_weight_T" + initializers.append(onh.from_array(weight_np.T, name=q_weight)) # pre-transposed [in, out] + + matmul_out = f"{prefix}_matmul" + nodes.append(oh.make_node("MatMul", inputs=[current, q_weight], outputs=[matmul_out])) + current = matmul_out + + if module._bias is not None: + bias_np = module._bias.detach().cpu().numpy().astype(np.float32) + if use_qonnx: + bias_fp_name = f"{prefix}_bias_fp" + initializers.append(onh.from_array(bias_np, name=bias_fp_name)) + k_b, i_b, f_b = module.bias_quantizer.get_quantization_bits() + b_nodes, q_bias = _quant_node( + f"{prefix}_bias", + bias_fp_name, + module.bias_quantizer.round_mode, + k_b, + i_b, + f_b, + initializers, + overflow_mode=getattr(module.bias_quantizer, "overflow", "SAT"), + ) + nodes.extend(b_nodes) + elif store_integer_weights: + k_b, i_b, f_b = module.bias_quantizer.get_quantization_bits() + b_nodes, q_bias = _int_weight_node(f"{prefix}_bias", bias_np, k_b, i_b, f_b, initializers) + nodes.extend(b_nodes) + else: + q_bias = f"{prefix}_bias" + initializers.append(onh.from_array(bias_np, name=q_bias)) + biased_out = f"{prefix}_biased" + nodes.append(oh.make_node("Add", inputs=[matmul_out, q_bias], outputs=[biased_out])) + current = biased_out + + current = _maybe_quant_output(module, prefix, current, nodes, initializers, quant_fn) + return current + + +def _add_dense(module, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights, integer_ops=False): + if integer_ops and not use_qonnx: + return _add_dense_integer(module, prefix, current, nodes, initializers) + current = _maybe_quant_input(module, prefix, current, nodes, initializers, quant_fn) + + weight_np = module._weight.detach().cpu().numpy().astype(np.float32) + if use_qonnx: + weight_fp_name = f"{prefix}_weight_fp" + initializers.append(onh.from_array(weight_np, name=weight_fp_name)) + k_w, i_w, f_w = module.weight_quantizer.get_quantization_bits() + w_nodes, q_weight = _quant_node( + f"{prefix}_weight", + weight_fp_name, + module.weight_quantizer.round_mode, + k_w, + i_w, + f_w, + initializers, + overflow_mode=getattr(module.weight_quantizer, "overflow", "SAT"), + ) + nodes.extend(w_nodes) + elif store_integer_weights: + k_w, i_w, f_w = module.weight_quantizer.get_quantization_bits() + w_nodes, q_weight = _int_weight_node(f"{prefix}_weight", weight_np, k_w, i_w, f_w, initializers) + nodes.extend(w_nodes) + else: + q_weight = f"{prefix}_weight" + initializers.append(onh.from_array(weight_np, name=q_weight)) + + # Use Gemm with transB=1 β€” weight stays in its native [out, in] layout, + # no Transpose node needed. Bias (if any) is fused as the third Gemm input. + gemm_inputs = [current, q_weight] + + if module._bias is not None: + bias_np = module._bias.detach().cpu().numpy().astype(np.float32) + if use_qonnx: + bias_fp_name = f"{prefix}_bias_fp" + initializers.append(onh.from_array(bias_np, name=bias_fp_name)) + k_b, i_b, f_b = module.bias_quantizer.get_quantization_bits() + b_nodes, q_bias = _quant_node( + f"{prefix}_bias", + bias_fp_name, + module.bias_quantizer.round_mode, + k_b, + i_b, + f_b, + initializers, + overflow_mode=getattr(module.bias_quantizer, "overflow", "SAT"), + ) + nodes.extend(b_nodes) + elif store_integer_weights: + k_b, i_b, f_b = module.bias_quantizer.get_quantization_bits() + b_nodes, q_bias = _int_weight_node(f"{prefix}_bias", bias_np, k_b, i_b, f_b, initializers) + nodes.extend(b_nodes) + else: + q_bias = f"{prefix}_bias" + initializers.append(onh.from_array(bias_np, name=q_bias)) + gemm_inputs.append(q_bias) + + gemm_out = f"{prefix}_gemm" + nodes.append(oh.make_node("Gemm", inputs=gemm_inputs, outputs=[gemm_out], transB=1)) + current = gemm_out + + current = _maybe_quant_output(module, prefix, current, nodes, initializers, quant_fn) + return current + + +def _add_conv(module, prefix, current, nodes, initializers, ndim, quant_fn, use_qonnx, store_integer_weights): + current = _maybe_quant_input(module, prefix, current, nodes, initializers, quant_fn) + + weight_np = module._weight.detach().cpu().numpy().astype(np.float32) + if use_qonnx: + weight_fp_name = f"{prefix}_weight_fp" + initializers.append(onh.from_array(weight_np, name=weight_fp_name)) + k_w, i_w, f_w = module.weight_quantizer.get_quantization_bits() + w_nodes, q_weight = _quant_node( + f"{prefix}_weight", + weight_fp_name, + module.weight_quantizer.round_mode, + k_w, + i_w, + f_w, + initializers, + overflow_mode=getattr(module.weight_quantizer, "overflow", "SAT"), + ) + nodes.extend(w_nodes) + elif store_integer_weights: + k_w, i_w, f_w = module.weight_quantizer.get_quantization_bits() + w_nodes, q_weight = _int_weight_node(f"{prefix}_weight", weight_np, k_w, i_w, f_w, initializers) + nodes.extend(w_nodes) + else: + q_weight = f"{prefix}_weight" + initializers.append(onh.from_array(weight_np, name=q_weight)) + + conv_inputs = [current, q_weight] + + if module._bias is not None: + bias_np = module._bias.detach().cpu().numpy().astype(np.float32) + if use_qonnx: + bias_fp_name = f"{prefix}_bias_fp" + initializers.append(onh.from_array(bias_np, name=bias_fp_name)) + k_b, i_b, f_b = module.bias_quantizer.get_quantization_bits() + b_nodes, q_bias = _quant_node( + f"{prefix}_bias", + bias_fp_name, + module.bias_quantizer.round_mode, + k_b, + i_b, + f_b, + initializers, + overflow_mode=getattr(module.bias_quantizer, "overflow", "SAT"), + ) + nodes.extend(b_nodes) + elif store_integer_weights: + k_b, i_b, f_b = module.bias_quantizer.get_quantization_bits() + b_nodes, q_bias = _int_weight_node(f"{prefix}_bias", bias_np, k_b, i_b, f_b, initializers) + nodes.extend(b_nodes) + else: + q_bias = f"{prefix}_bias" + initializers.append(onh.from_array(bias_np, name=q_bias)) + conv_inputs.append(q_bias) + + padding = module.padding + if isinstance(padding, str): + auto_pad = "SAME_UPPER" if padding == "same" else "VALID" + pads = None + else: + auto_pad = "NOTSET" + pads = _torch_padding_to_onnx(padding, ndim) + + to_list = lambda v, n: list(v) if hasattr(v, "__iter__") else [v] * n # noqa: E731 + conv_attrs = dict( + kernel_shape=to_list(module.kernel_size, ndim), + strides=to_list(module.stride, ndim), + dilations=to_list(module.dilation, ndim), + group=module.groups, + auto_pad=auto_pad, + ) + if pads is not None: + conv_attrs["pads"] = pads + + conv_out = f"{prefix}_conv" + nodes.append(oh.make_node("Conv", inputs=conv_inputs, outputs=[conv_out], **conv_attrs)) + current = conv_out + + current = _maybe_quant_output(module, prefix, current, nodes, initializers, quant_fn) + return current + + +def _add_batchnorm(module, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights): + current = _maybe_quant_input(module, prefix, current, nodes, initializers, quant_fn) + + gamma_np = module._weight.detach().cpu().numpy().astype(np.float32) + beta_np = module._bias.detach().cpu().numpy().astype(np.float32) + + if use_qonnx: + gamma_fp_name = f"{prefix}_gamma_fp" + initializers.append(onh.from_array(gamma_np, name=gamma_fp_name)) + k_w, i_w, f_w = module.weight_quantizer.get_quantization_bits() + g_nodes, q_gamma = _quant_node( + f"{prefix}_gamma", + gamma_fp_name, + module.weight_quantizer.round_mode, + k_w, + i_w, + f_w, + initializers, + overflow_mode=getattr(module.weight_quantizer, "overflow", "SAT"), + ) + nodes.extend(g_nodes) + + beta_fp_name = f"{prefix}_beta_fp" + initializers.append(onh.from_array(beta_np, name=beta_fp_name)) + k_b, i_b, f_b = module.bias_quantizer.get_quantization_bits() + b_nodes, q_beta = _quant_node( + f"{prefix}_beta", + beta_fp_name, + module.bias_quantizer.round_mode, + k_b, + i_b, + f_b, + initializers, + overflow_mode=getattr(module.bias_quantizer, "overflow", "SAT"), + ) + nodes.extend(b_nodes) + elif store_integer_weights: + k_w, i_w, f_w = module.weight_quantizer.get_quantization_bits() + g_nodes, q_gamma = _int_weight_node(f"{prefix}_gamma", gamma_np, k_w, i_w, f_w, initializers) + nodes.extend(g_nodes) + k_b, i_b, f_b = module.bias_quantizer.get_quantization_bits() + b_nodes, q_beta = _int_weight_node(f"{prefix}_beta", beta_np, k_b, i_b, f_b, initializers) + nodes.extend(b_nodes) + else: + q_gamma = f"{prefix}_gamma" + q_beta = f"{prefix}_beta" + initializers.append(onh.from_array(gamma_np, name=q_gamma)) + initializers.append(onh.from_array(beta_np, name=q_beta)) + + mean_name = f"{prefix}_running_mean" + var_name = f"{prefix}_running_var" + initializers.append(onh.from_array(module.running_mean.detach().cpu().numpy().astype(np.float32), name=mean_name)) + initializers.append(onh.from_array(module.running_var.detach().cpu().numpy().astype(np.float32), name=var_name)) + + bn_out = f"{prefix}_bn" + nodes.append( + oh.make_node( + "BatchNormalization", + inputs=[current, q_gamma, q_beta, mean_name, var_name], + outputs=[bn_out], + epsilon=float(module.eps), + ) + ) + return bn_out + + +def _add_layernorm(module, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights): + """PQLayerNorm. Emits LayerNormalization (opset >= 17 required).""" + current = _maybe_quant_input(module, prefix, current, nodes, initializers, quant_fn) + + ns = ( + tuple(int(d) for d in module.normalized_shape) + if hasattr(module.normalized_shape, "__iter__") + else (int(module.normalized_shape),) + ) + axis = -len(ns) + + has_weight = module._weight is not None + has_bias = module._bias is not None + + gamma_np = module._weight.detach().cpu().numpy().astype(np.float32) if has_weight else np.ones(ns, dtype=np.float32) + beta_np = module._bias.detach().cpu().numpy().astype(np.float32) if has_bias else None + + if use_qonnx and has_weight: + gamma_fp_name = f"{prefix}_gamma_fp" + initializers.append(onh.from_array(gamma_np, name=gamma_fp_name)) + k_w, i_w, f_w = module.weight_quantizer.get_quantization_bits() + g_nodes, q_gamma = _quant_node( + f"{prefix}_gamma", + gamma_fp_name, + module.weight_quantizer.round_mode, + k_w, + i_w, + f_w, + initializers, + overflow_mode=getattr(module.weight_quantizer, "overflow", "SAT"), + ) + nodes.extend(g_nodes) + if has_bias: + beta_fp_name = f"{prefix}_beta_fp" + initializers.append(onh.from_array(beta_np, name=beta_fp_name)) + k_b, i_b, f_b = module.bias_quantizer.get_quantization_bits() + b_nodes, q_beta = _quant_node( + f"{prefix}_beta", + beta_fp_name, + module.bias_quantizer.round_mode, + k_b, + i_b, + f_b, + initializers, + overflow_mode=getattr(module.bias_quantizer, "overflow", "SAT"), + ) + nodes.extend(b_nodes) + elif store_integer_weights and has_weight: + k_w, i_w, f_w = module.weight_quantizer.get_quantization_bits() + g_nodes, q_gamma = _int_weight_node(f"{prefix}_gamma", gamma_np, k_w, i_w, f_w, initializers) + nodes.extend(g_nodes) + if has_bias: + k_b, i_b, f_b = module.bias_quantizer.get_quantization_bits() + b_nodes, q_beta = _int_weight_node(f"{prefix}_beta", beta_np, k_b, i_b, f_b, initializers) + nodes.extend(b_nodes) + else: + q_gamma = f"{prefix}_gamma" + initializers.append(onh.from_array(gamma_np, name=q_gamma)) + if has_bias: + q_beta = f"{prefix}_beta" + initializers.append(onh.from_array(beta_np, name=q_beta)) + + ln_inputs = [current, q_gamma] + if has_bias: + ln_inputs.append(q_beta) + ln_out = f"{prefix}_ln" + nodes.append( + oh.make_node( + "LayerNormalization", + inputs=ln_inputs, + outputs=[ln_out], + axis=axis, + epsilon=float(module.eps), + ) + ) + current = ln_out + current = _maybe_quant_output(module, prefix, current, nodes, initializers, quant_fn) + return current + + +def _add_avgpool(module, prefix, current, nodes, initializers, ndim, quant_fn): + current = _maybe_quant_input(module, prefix, current, nodes, initializers, quant_fn) + + to_list = lambda v, n: list(v) if hasattr(v, "__iter__") else [v] * n # noqa: E731 + pool_out = f"{prefix}_pool" + nodes.append( + oh.make_node( + "AveragePool", + inputs=[current], + outputs=[pool_out], + kernel_shape=to_list(module.kernel_size, ndim), + strides=to_list(module.stride, ndim), + pads=_torch_padding_to_onnx(module.padding, ndim), + ceil_mode=int(module.ceil_mode), + count_include_pad=int(module.count_include_pad), + ) + ) + current = pool_out + + current = _maybe_quant_output(module, prefix, current, nodes, initializers, quant_fn) + return current + + +# --------------------------------------------------------------------------- +# multi-head attention graph builder +# --------------------------------------------------------------------------- + + +def _add_mha(module, prefix, q_input, k_input, v_input, nodes, initializers, quant_fn, use_qonnx, store_integer_weights): + """Build ONNX nodes for PQMultiheadAttention. + + Decomposes multi-head attention into primitive ONNX ops: + + [optional transpose if not batch_first] + Q/K/V Gemm projections + Reshape (B, L, E) β†’ (B, H, L, head_dim) + Transpose + MatMul(Q, K^T) * scale β†’ optional Quant + Softmax β†’ optional Quant + MatMul(attn_weights, V) β†’ optional Quant + Transpose + Reshape (B, T, E) + out_proj Gemm + [optional transpose back if not batch_first] + + Returns (out_name, avg_attn_weights_name): the projected output and the + attention weights averaged over heads (B, T, S). Both names are valid ONNX + value names so downstream getitem(mha, 0) / getitem(mha, 1) work in the FX + converter. + + Note: if ``approximate_softmax=True`` the module uses a polynomial + approximation in PyTorch, but ONNX has no equivalent standard op β€” a plain + ``Softmax`` node is emitted instead. + """ + H = module.num_heads + head_dim = module.head_dim + E = module.embed_dim + scale_val = float(module.scale) + + # --- Optional transpose for seq-first inputs (T, B, E) β†’ (B, T, E) --- + if not module.batch_first: + q_t = f"{prefix}_q_in_t" + k_t = f"{prefix}_k_in_t" + v_t = f"{prefix}_v_in_t" + nodes.append(oh.make_node("Transpose", inputs=[q_input], outputs=[q_t], perm=[1, 0, 2])) + nodes.append(oh.make_node("Transpose", inputs=[k_input], outputs=[k_t], perm=[1, 0, 2])) + nodes.append(oh.make_node("Transpose", inputs=[v_input], outputs=[v_t], perm=[1, 0, 2])) + q_input, k_input, v_input = q_t, k_t, v_t + + # --- Q / K / V projections: (B, L, E) β†’ (B, L, E) via MatMul (input is rank-3) --- + q_proj_out = _add_dense_nd( + module.q_proj, f"{prefix}_q_proj", q_input, nodes, initializers, quant_fn, use_qonnx, store_integer_weights + ) + k_proj_out = _add_dense_nd( + module.k_proj, f"{prefix}_k_proj", k_input, nodes, initializers, quant_fn, use_qonnx, store_integer_weights + ) + v_proj_out = _add_dense_nd( + module.v_proj, f"{prefix}_v_proj", v_input, nodes, initializers, quant_fn, use_qonnx, store_integer_weights + ) + + # --- Helper: (B, L, E) β†’ (B, H, L, head_dim) using dynamic shapes --- + def _split_heads(x_name, pfx): + shape_out = f"{pfx}_shape" + b_scalar = f"{pfx}_b_sc" + l_scalar = f"{pfx}_l_sc" + b_1d = f"{pfx}_b_1d" + l_1d = f"{pfx}_l_1d" + h_1d_const = f"{pfx}_H_1d" + hd_1d_const = f"{pfx}_hd_1d" + shape_4d = f"{pfx}_shape4d" + reshaped = f"{pfx}_reshaped" + transposed = f"{pfx}_transposed" + idx0 = f"{pfx}_gi0" + idx1 = f"{pfx}_gi1" + ax0 = f"{pfx}_ax0" + + nodes.append(oh.make_node("Shape", inputs=[x_name], outputs=[shape_out])) + initializers.extend( + [ + onh.from_array(np.array(0, dtype=np.int64), name=idx0), + onh.from_array(np.array(1, dtype=np.int64), name=idx1), + onh.from_array(np.array([0], dtype=np.int64), name=ax0), + onh.from_array(np.array([H], dtype=np.int64), name=h_1d_const), + onh.from_array(np.array([head_dim], dtype=np.int64), name=hd_1d_const), + ] + ) + nodes.append(oh.make_node("Gather", inputs=[shape_out, idx0], outputs=[b_scalar])) + nodes.append(oh.make_node("Gather", inputs=[shape_out, idx1], outputs=[l_scalar])) + nodes.append(oh.make_node("Unsqueeze", inputs=[b_scalar, ax0], outputs=[b_1d])) + nodes.append(oh.make_node("Unsqueeze", inputs=[l_scalar, ax0], outputs=[l_1d])) + nodes.append(oh.make_node("Concat", inputs=[b_1d, l_1d, h_1d_const, hd_1d_const], outputs=[shape_4d], axis=0)) + nodes.append(oh.make_node("Reshape", inputs=[x_name, shape_4d], outputs=[reshaped])) + # (B, L, H, head_dim) β†’ (B, H, L, head_dim) + nodes.append(oh.make_node("Transpose", inputs=[reshaped], outputs=[transposed], perm=[0, 2, 1, 3])) + return transposed + + q_h = _split_heads(q_proj_out, f"{prefix}_q") + k_h = _split_heads(k_proj_out, f"{prefix}_k") + v_h = _split_heads(v_proj_out, f"{prefix}_v") + + # --- k^T: (B, H, S, head_dim) β†’ (B, H, head_dim, S) --- + k_t_name = f"{prefix}_k_T" + nodes.append(oh.make_node("Transpose", inputs=[k_h], outputs=[k_t_name], perm=[0, 1, 3, 2])) + + # --- Scaled dot-product scores: (B, H, T, head_dim) @ (B, H, head_dim, S) β†’ (B, H, T, S) --- + raw_scores = f"{prefix}_scores_raw" + scaled_scores = f"{prefix}_scores_scaled" + scale_cst = f"{prefix}_attn_scale" + nodes.append(oh.make_node("MatMul", inputs=[q_h, k_t_name], outputs=[raw_scores])) + initializers.append(onh.from_array(np.array(scale_val, dtype=np.float32), name=scale_cst)) + nodes.append(oh.make_node("Mul", inputs=[raw_scores, scale_cst], outputs=[scaled_scores])) + current = scaled_scores + + # --- Optional attn-score quantization --- + if ( + getattr(module, "quantize_attn_scores", False) + and hasattr(module, "attn_score_quantizer") + and getattr(module, "enable_quantization", True) + ): + q = module.attn_score_quantizer + k_q, i_q, f_q = q.get_quantization_bits() + q_nodes, current = quant_fn( + f"{prefix}_attn_score_q", + current, + q.round_mode, + k_q, + i_q, + f_q, + initializers, + overflow_mode=getattr(q, "overflow", "SAT"), + ) + nodes.extend(q_nodes) + + # --- Softmax (dim=-1); approximate_softmax falls back to standard Softmax in ONNX --- + attn_w_name = f"{prefix}_attn_weights" + nodes.append(oh.make_node("Softmax", inputs=[current], outputs=[attn_w_name], axis=-1)) + current = attn_w_name + + # --- Optional attn-weight quantization --- + if ( + getattr(module, "quantize_attn_weights", False) + and hasattr(module, "attn_weight_quantizer") + and getattr(module, "enable_quantization", True) + ): + q = module.attn_weight_quantizer + k_q, i_q, f_q = q.get_quantization_bits() + q_nodes, current = quant_fn( + f"{prefix}_attn_weight_q", + current, + q.round_mode, + k_q, + i_q, + f_q, + initializers, + overflow_mode=getattr(q, "overflow", "SAT"), + ) + nodes.extend(q_nodes) + + # --- Context: (B, H, T, S) @ (B, H, S, head_dim) β†’ (B, H, T, head_dim) --- + ctx_raw = f"{prefix}_ctx_raw" + nodes.append(oh.make_node("MatMul", inputs=[current, v_h], outputs=[ctx_raw])) + current_ctx = ctx_raw + + # --- Optional context quantization --- + if ( + getattr(module, "quantize_context", False) + and hasattr(module, "context_quantizer") + and getattr(module, "enable_quantization", True) + ): + q = module.context_quantizer + k_q, i_q, f_q = q.get_quantization_bits() + q_nodes, current_ctx = quant_fn( + f"{prefix}_context_q", + current_ctx, + q.round_mode, + k_q, + i_q, + f_q, + initializers, + overflow_mode=getattr(q, "overflow", "SAT"), + ) + nodes.extend(q_nodes) + + # --- Merge heads: (B, H, T, head_dim) β†’ (B, T, E) using dynamic shapes --- + ctx_t = f"{prefix}_ctx_t" # after Transpose β†’ (B, T, H, head_dim) + ctx_shape = f"{prefix}_ctx_shape" + ctx_b_sc = f"{prefix}_ctx_b_sc" + ctx_t_sc = f"{prefix}_ctx_t_sc" + ctx_b_1d = f"{prefix}_ctx_b_1d" + ctx_t_1d = f"{prefix}_ctx_t_1d" + ctx_E_1d = f"{prefix}_ctx_E_1d" + ctx_ax0 = f"{prefix}_ctx_ax0" + ctx_gi0 = f"{prefix}_ctx_gi0" + ctx_gi1 = f"{prefix}_ctx_gi1" + ctx_3d = f"{prefix}_ctx_shape3d" + ctx_merged = f"{prefix}_ctx_merged" + + nodes.append(oh.make_node("Transpose", inputs=[current_ctx], outputs=[ctx_t], perm=[0, 2, 1, 3])) + nodes.append(oh.make_node("Shape", inputs=[ctx_t], outputs=[ctx_shape])) + initializers += [ + onh.from_array(np.array(0, dtype=np.int64), name=ctx_gi0), + onh.from_array(np.array(1, dtype=np.int64), name=ctx_gi1), + onh.from_array(np.array([0], dtype=np.int64), name=ctx_ax0), + onh.from_array(np.array([E], dtype=np.int64), name=ctx_E_1d), + ] + nodes.append(oh.make_node("Gather", inputs=[ctx_shape, ctx_gi0], outputs=[ctx_b_sc])) + nodes.append(oh.make_node("Gather", inputs=[ctx_shape, ctx_gi1], outputs=[ctx_t_sc])) + nodes.append(oh.make_node("Unsqueeze", inputs=[ctx_b_sc, ctx_ax0], outputs=[ctx_b_1d])) + nodes.append(oh.make_node("Unsqueeze", inputs=[ctx_t_sc, ctx_ax0], outputs=[ctx_t_1d])) + nodes.append(oh.make_node("Concat", inputs=[ctx_b_1d, ctx_t_1d, ctx_E_1d], outputs=[ctx_3d], axis=0)) + nodes.append(oh.make_node("Reshape", inputs=[ctx_t, ctx_3d], outputs=[ctx_merged])) + + # --- Output projection (rank-3 input: (B, T, E)) --- + out = _add_dense_nd( + module.out_proj, f"{prefix}_out_proj", ctx_merged, nodes, initializers, quant_fn, use_qonnx, store_integer_weights + ) + + # --- Average attention weights over heads: (B, H, T, S) β†’ (B, T, S) --- + # Emitted so that getitem(mha, 1) has a valid ONNX value name. + avg_attn = f"{prefix}_avg_attn_weights" + nodes.append(oh.make_node("ReduceMean", inputs=[attn_w_name], outputs=[avg_attn], axes=[1], keepdims=0)) + + # --- Optional transpose back for seq-first output --- + if not module.batch_first: + out_final = f"{prefix}_out_seq_first" + nodes.append(oh.make_node("Transpose", inputs=[out], outputs=[out_final], perm=[1, 0, 2])) + return out_final, avg_attn + + return out, avg_attn + + +# --------------------------------------------------------------------------- +# shared module dispatch (used by both sequential and FX converters) +# --------------------------------------------------------------------------- + + +def _emit_module( + module, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights, integer_ops=False +): + """Emit ONNX nodes for a single PQuant or standard torch.nn module.""" + if isinstance(module, PQDense): + return _add_dense( + module, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights, integer_ops + ) + if isinstance(module, PQConv2d): + return _add_conv( + module, + prefix, + current, + nodes, + initializers, + ndim=2, + quant_fn=quant_fn, + use_qonnx=use_qonnx, + store_integer_weights=store_integer_weights, + ) + if isinstance(module, PQConv1d): + return _add_conv( + module, + prefix, + current, + nodes, + initializers, + ndim=1, + quant_fn=quant_fn, + use_qonnx=use_qonnx, + store_integer_weights=store_integer_weights, + ) + if isinstance(module, (PQBatchNorm2d, PQBatchNorm1d)): + return _add_batchnorm(module, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights) + if isinstance(module, PQLayerNorm): + return _add_layernorm(module, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights) + if isinstance(module, PQAvgPool2d): + return _add_avgpool(module, prefix, current, nodes, initializers, ndim=2, quant_fn=quant_fn) + if isinstance(module, PQAvgPool1d): + return _add_avgpool(module, prefix, current, nodes, initializers, ndim=1, quant_fn=quant_fn) + if isinstance(module, nn.ReLU): + out = f"{prefix}_relu" + nodes.append(oh.make_node("Relu", inputs=[current], outputs=[out])) + return out + if isinstance(module, nn.Flatten): + out = f"{prefix}_flatten" + nodes.append(oh.make_node("Flatten", inputs=[current], outputs=[out], axis=module.start_dim)) + return out + if isinstance(module, (nn.BatchNorm1d, nn.BatchNorm2d)): + gamma_name = f"{prefix}_bn_gamma" + beta_name = f"{prefix}_bn_beta" + mean_name = f"{prefix}_bn_mean" + var_name = f"{prefix}_bn_var" + initializers += [ + onh.from_array(module.weight.detach().cpu().numpy().astype(np.float32), name=gamma_name), + onh.from_array(module.bias.detach().cpu().numpy().astype(np.float32), name=beta_name), + onh.from_array(module.running_mean.detach().cpu().numpy().astype(np.float32), name=mean_name), + onh.from_array(module.running_var.detach().cpu().numpy().astype(np.float32), name=var_name), + ] + out = f"{prefix}_bn" + nodes.append( + oh.make_node( + "BatchNormalization", + inputs=[current, gamma_name, beta_name, mean_name, var_name], + outputs=[out], + epsilon=float(module.eps), + ) + ) + return out + if isinstance(module, (nn.Dropout, nn.Dropout2d)): + return current # identity at inference + if isinstance(module, nn.LeakyReLU): + out = f"{prefix}_leakyrelu" + nodes.append(oh.make_node("LeakyRelu", inputs=[current], outputs=[out], alpha=module.negative_slope)) + return out + if isinstance(module, nn.MaxPool2d): + out = f"{prefix}_maxpool" + kernel = module.kernel_size if isinstance(module.kernel_size, (list, tuple)) else [module.kernel_size] * 2 + stride = module.stride if isinstance(module.stride, (list, tuple)) else [module.stride] * 2 + pad = module.padding if isinstance(module.padding, (list, tuple)) else [module.padding] * 2 + nodes.append( + oh.make_node( + "MaxPool", + inputs=[current], + outputs=[out], + kernel_shape=list(kernel), + strides=list(stride), + pads=[pad[0], pad[1], pad[0], pad[1]], + ) + ) + return out + if isinstance(module, nn.Upsample): + # Emit a Resize node with nearest/bilinear mode and scale factors. + roi_name = f"{prefix}_upsample_roi" + scales_name = f"{prefix}_upsample_scales" + initializers.append(onh.from_array(np.array([], dtype=np.float32), name=roi_name)) + scale_factor = module.scale_factor + if isinstance(scale_factor, (int, float)): + scale_factor = (scale_factor, scale_factor) + scales = np.array([1.0, 1.0, float(scale_factor[0]), float(scale_factor[1])], dtype=np.float32) + initializers.append(onh.from_array(scales, name=scales_name)) + mode = "nearest" if module.mode == "nearest" else "linear" + out = f"{prefix}_upsample" + nodes.append( + oh.make_node( + "Resize", + inputs=[current, roi_name, scales_name], + outputs=[out], + mode=mode, + coordinate_transformation_mode="asymmetric", + ) + ) + return out + if isinstance(module, PQActivation): + current = _maybe_quant_input(module, prefix, current, nodes, initializers, quant_fn) + act = module.activation_name + act_out = f"{prefix}_act" + if act == "relu": + nodes.append(oh.make_node("Relu", inputs=[current], outputs=[act_out])) + elif act == "tanh": + nodes.append(oh.make_node("Tanh", inputs=[current], outputs=[act_out])) + elif act == "hard_tanh": + cmin_name = f"{prefix}_htanh_min" + cmax_name = f"{prefix}_htanh_max" + initializers += [ + onh.from_array(np.array(-1.0, dtype=np.float32), name=cmin_name), + onh.from_array(np.array(1.0, dtype=np.float32), name=cmax_name), + ] + nodes.append(oh.make_node("Clip", inputs=[current, cmin_name, cmax_name], outputs=[act_out])) + elif act == "leaky_relu": + nodes.append( + oh.make_node( + "LeakyRelu", inputs=[current], outputs=[act_out], alpha=module.activation_function.negative_slope + ) + ) + elif act == "gelu": + # Decompose so the default opset (13) works; ONNX added a Gelu op only in opset 20. + approximate = getattr(module.activation_function, "approximate", "none") + half_name = f"{prefix}_gelu_half" + one_name = f"{prefix}_gelu_one" + initializers += [ + onh.from_array(np.array(0.5, dtype=np.float32), name=half_name), + onh.from_array(np.array(1.0, dtype=np.float32), name=one_name), + ] + if approximate == "tanh": + # 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))) + c0_name = f"{prefix}_gelu_sqrt2_over_pi" + c1_name = f"{prefix}_gelu_c1" + three_name = f"{prefix}_gelu_three" + initializers += [ + onh.from_array(np.array(np.sqrt(2.0 / np.pi), dtype=np.float32), name=c0_name), + onh.from_array(np.array(0.044715, dtype=np.float32), name=c1_name), + onh.from_array(np.array(3.0, dtype=np.float32), name=three_name), + ] + x3 = f"{prefix}_gelu_x3" + cx3 = f"{prefix}_gelu_cx3" + inner = f"{prefix}_gelu_inner" + scaled = f"{prefix}_gelu_scaled" + tanh_out = f"{prefix}_gelu_tanh" + plus_one = f"{prefix}_gelu_plus1" + x_times = f"{prefix}_gelu_xprod" + nodes += [ + oh.make_node("Pow", inputs=[current, three_name], outputs=[x3]), + oh.make_node("Mul", inputs=[x3, c1_name], outputs=[cx3]), + oh.make_node("Add", inputs=[current, cx3], outputs=[inner]), + oh.make_node("Mul", inputs=[inner, c0_name], outputs=[scaled]), + oh.make_node("Tanh", inputs=[scaled], outputs=[tanh_out]), + oh.make_node("Add", inputs=[tanh_out, one_name], outputs=[plus_one]), + oh.make_node("Mul", inputs=[current, plus_one], outputs=[x_times]), + oh.make_node("Mul", inputs=[x_times, half_name], outputs=[act_out]), + ] + else: + # Exact: 0.5 * x * (1 + erf(x / sqrt(2))) + inv_sqrt2_name = f"{prefix}_gelu_inv_sqrt2" + initializers.append(onh.from_array(np.array(1.0 / np.sqrt(2.0), dtype=np.float32), name=inv_sqrt2_name)) + scaled = f"{prefix}_gelu_scaled" + erf_out = f"{prefix}_gelu_erf" + plus_one = f"{prefix}_gelu_plus1" + x_times = f"{prefix}_gelu_xprod" + nodes += [ + oh.make_node("Mul", inputs=[current, inv_sqrt2_name], outputs=[scaled]), + oh.make_node("Erf", inputs=[scaled], outputs=[erf_out]), + oh.make_node("Add", inputs=[erf_out, one_name], outputs=[plus_one]), + oh.make_node("Mul", inputs=[current, plus_one], outputs=[x_times]), + oh.make_node("Mul", inputs=[x_times, half_name], outputs=[act_out]), + ] + else: + raise TypeError(f"PQActivation: unsupported activation {act!r} for ONNX export") + current = act_out + current = _maybe_quant_output(module, prefix, current, nodes, initializers, quant_fn) + return current + if isinstance(module, PQMultiheadAttention): + # Sequential converter: treat as self-attention (Q = K = V = current). + # Returns (out_name, avg_attn_name); expose only the attention output. + out, _ = _add_mha( + module, prefix, current, current, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights + ) + return out + raise TypeError(f"Unsupported module type for ONNX export: {type(module).__name__}") + + +# --------------------------------------------------------------------------- +# main conversion +# --------------------------------------------------------------------------- + + +def convert_to_onnx( + model: nn.Sequential, + input_shape: tuple, + output_path: str = "model.onnx", + opset: int = 13, + use_qonnx: bool = False, + store_integer_weights: bool = False, + integer_ops: bool = False, + include_clip: bool = True, + batch_size: int | None = None, +) -> onnx.ModelProto: + """ + Convert a Sequential model of PQuant layers to ONNX or QONNX. + + Args: + model: Trained nn.Sequential. Call apply_final_compression() + on all PQ modules before passing here. + input_shape: Shape of a single sample (excluding batch), e.g. (3, 32, 32). + output_path: Where to save the .onnx file. + opset: ONNX opset version (β‰₯13 required for per-channel DequantizeLinear). + use_qonnx: If True, emit QONNX Quant custom nodes (requires qonnx runtime). + If False (default), emit Clip+QuantizeLinear+DequantizeLinear + nodes runnable with plain onnxruntime. + store_integer_weights: If True (and use_qonnx=False), store weight/bias initializers + as int8/uint8 followed by DequantizeLinear instead of float32. + Ignored when use_qonnx=True or integer_ops=True. + integer_ops: If True (and use_qonnx=False), use MatMulInteger for Dense layers + so the inner product runs in int32 arithmetic. Weights are stored + as int8 (pre-transposed) and a single DequantizeLinear converts the + int32 accumulator back to float using the combined scale s_x * s_w. + Implies integer weight storage; store_integer_weights is ignored. + include_clip: Prepend a Clip node before each QuantizeLinear when True (default). + Set to False to emit bare QuantizeLinear+DequantizeLinear pairs β€” + safe when values are guaranteed in-range at inference time since + QuantizeLinear saturates naturally. Ignored when use_qonnx=True. + batch_size: If not None, fix the batch dimension of all graph inputs and + outputs to this value. If None (default), the batch dimension + is left dynamic. + + Returns: + The constructed onnx.ModelProto. + """ + model.eval() + quant_fn = _quant_node if use_qonnx else functools.partial(_qdq_node, include_clip=include_clip) + + nodes: list[onnx.NodeProto] = [] + initializers: list[onnx.TensorProto] = [] + current = "input" + + for layer_idx, module in enumerate(model): + prefix = f"layer{layer_idx}" + current = _emit_module( + module, prefix, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights, integer_ops + ) + + with torch.no_grad(): + dummy_out = model(torch.zeros(1, *input_shape)) + batch_dim = batch_size # None β†’ dynamic, int β†’ fixed + output_shape = [batch_dim] + list(dummy_out.shape[1:]) + + batch_dim_vi = oh.make_tensor_value_info("input", TensorProto.FLOAT, [batch_dim, *input_shape]) + output_vi = oh.make_tensor_value_info(current, TensorProto.FLOAT, output_shape) + + graph = oh.make_graph( + nodes=nodes, + name="pquant_onnx", + inputs=[batch_dim_vi], + outputs=[output_vi], + initializer=initializers, + ) + + opset_imports = [oh.make_opsetid("", opset)] + if use_qonnx: + opset_imports.append(oh.make_opsetid("qonnx.custom_op.general", 1)) + model_proto = oh.make_model(graph, opset_imports=opset_imports) + model_proto.ir_version = 6 + + onnx.checker.check_model(model_proto) + onnx.save(model_proto, output_path) + fmt = "QONNX" if use_qonnx else "ONNX (QDQ)" + logging.info("Saved %s model β†’ %s", fmt, output_path) + return model_proto + + +# --------------------------------------------------------------------------- +# Hardware-targeted static-QDQ LayerNormalization graph +# --------------------------------------------------------------------------- + + +def _is_pow2(n: int) -> bool: + return n > 0 and (n & (n - 1)) == 0 + + +def export_qdq_layernorm( + output_path: str, + input_shape, + gamma: np.ndarray, + beta: np.ndarray, + input_scale_log2: int, + output_scale_log2: int, + eps_q0: int = 1, + opset: int = 17, +) -> onnx.ModelProto: + """Build and save a single-LayerNormalization ONNX graph using static QDQ quantization. + + Graph layout:: + + int8 input -> DequantizeLinear -> LayerNormalization -> QuantizeLinear -> DequantizeLinear -> output + + All quantization parameters are explicit float32 initializers (no dynamic + tensors). Per-tensor quantization only; activation zero-points are 0. + + Constraints (validated at build time, not in the graph): + * ``input_shape`` is rank-2 or rank-3 with no dynamic dims. + * Last dim ``D`` is a power of two AND a multiple of 32. + * ``gamma``/``beta`` are 1-D float arrays of length ``D``. + * ``gamma`` is exactly representable as int16 with scale ``2**-7`` (Q7). + * ``beta`` is exactly representable as int16 with scale ``2**-15`` (Q15). + * Input/output scales are exact powers of two, given as log2 exponents. + * ``epsilon = eps_q0 * input_scale**2`` with integer ``eps_q0 >= 1``. + * Normalization axis is the last axis. + + Args: + output_path: Where to save the .onnx file. + input_shape: Static shape of the int8 graph input, e.g. ``(4, 64)`` or ``(1, 4, 64)``. + gamma: Constant gamma initializer, shape ``(D,)``. + beta: Constant beta initializer, shape ``(D,)``. + input_scale_log2: Integer ``a`` with input scale ``= 2**a``. + output_scale_log2: Integer ``b`` with output scale ``= 2**b``. + eps_q0: Positive integer ``>= 1``; ``epsilon = eps_q0 * (2**a)**2``. + opset: ONNX opset version (must be ``>= 17`` for LayerNormalization). + + Returns: + The constructed ``onnx.ModelProto``. + """ + # ----- validate shape ----- + input_shape = tuple(int(d) for d in input_shape) + if len(input_shape) not in (2, 3): + raise ValueError(f"input_shape rank must be 2 or 3, got {len(input_shape)} ({input_shape})") + for d in input_shape: + if d <= 0: + raise ValueError(f"input_shape must be fully static and positive, got {input_shape}") + D = input_shape[-1] + if not _is_pow2(D): + raise ValueError(f"last dim must be a power of two, got {D}") + if D % 32 != 0: + raise ValueError(f"last dim must be a multiple of 32, got {D}") + + # ----- validate gamma / beta ----- + gamma = np.asarray(gamma, dtype=np.float32) + beta = np.asarray(beta, dtype=np.float32) + if gamma.shape != (D,): + raise ValueError(f"gamma must have shape ({D},), got {gamma.shape}") + if beta.shape != (D,): + raise ValueError(f"beta must have shape ({D},), got {beta.shape}") + + GAMMA_F = 7 # Q7 in int16 -> scale = 2**-7 + BETA_F = 15 # Q15 in int16 -> scale = 2**-15 + INT16_MIN, INT16_MAX = -(2**15), 2**15 - 1 + + def _check_q_int16(arr: np.ndarray, frac_bits: int, name: str) -> None: + scaled = arr.astype(np.float64) * (2**frac_bits) + rounded = np.round(scaled) + # Exactly representable: rounding is a no-op (within fp slack). + if not np.allclose(scaled, rounded, atol=1e-4): + raise ValueError( + f"{name} not exactly representable as int16 Q{frac_bits} " + f"(max abs round error = {np.max(np.abs(scaled - rounded)):.6g})" + ) + if rounded.min() < INT16_MIN or rounded.max() > INT16_MAX: + raise ValueError(f"{name} overflows int16 at Q{frac_bits} " f"(range [{rounded.min()}, {rounded.max()}])") + + _check_q_int16(gamma, GAMMA_F, "gamma") + _check_q_int16(beta, BETA_F, "beta") + + # ----- validate quant params ----- + input_scale_log2 = int(input_scale_log2) + output_scale_log2 = int(output_scale_log2) + eps_q0 = int(eps_q0) + if eps_q0 < 1: + raise ValueError(f"eps_q0 must be >= 1, got {eps_q0}") + + if opset < 17: + raise ValueError(f"opset must be >= 17 for LayerNormalization, got {opset}") + + input_scale = float(2.0**input_scale_log2) + output_scale = float(2.0**output_scale_log2) + epsilon = float(eps_q0) * input_scale * input_scale + + # ----- build initializers ----- + initializers = [ + onh.from_array(np.array(input_scale, dtype=np.float32), name="input_scale"), + onh.from_array(np.array(0, dtype=np.int8), name="input_zero_point"), + onh.from_array(np.array(output_scale, dtype=np.float32), name="output_scale"), + onh.from_array(np.array(0, dtype=np.int8), name="output_zero_point"), + onh.from_array(gamma.astype(np.float32), name="gamma"), + onh.from_array(beta.astype(np.float32), name="beta"), + ] + + # ----- build nodes ----- + nodes = [ + oh.make_node( + "DequantizeLinear", + inputs=["input_q", "input_scale", "input_zero_point"], + outputs=["x_dq"], + name="input_dq", + ), + oh.make_node( + "LayerNormalization", + inputs=["x_dq", "gamma", "beta"], + outputs=["ln_out"], + name="layernorm", + axis=-1, + epsilon=epsilon, + ), + oh.make_node( + "QuantizeLinear", + inputs=["ln_out", "output_scale", "output_zero_point"], + outputs=["y_q"], + name="output_q", + ), + oh.make_node( + "DequantizeLinear", + inputs=["y_q", "output_scale", "output_zero_point"], + outputs=["output"], + name="output_dq", + ), + ] + + # ----- build graph + model ----- + input_vi = oh.make_tensor_value_info("input_q", TensorProto.INT8, list(input_shape)) + output_vi = oh.make_tensor_value_info("output", TensorProto.FLOAT, list(input_shape)) + + graph = oh.make_graph( + nodes=nodes, + name="qdq_layernorm", + inputs=[input_vi], + outputs=[output_vi], + initializer=initializers, + ) + + model_proto = oh.make_model(graph, opset_imports=[oh.make_opsetid("", opset)]) + model_proto.ir_version = 8 + + # Strip any initializer names that the onnx library may have added to graph.input. + _init_names = {t.name for t in model_proto.graph.initializer} + _data_inputs = [vi for vi in model_proto.graph.input if vi.name not in _init_names] + del model_proto.graph.input[:] + model_proto.graph.input.extend(_data_inputs) + + onnx.checker.check_model(model_proto) + onnx.save(model_proto, output_path) + return model_proto + + +# --------------------------------------------------------------------------- +# FX-based conversion (supports arbitrary nn.Module topology / skip connections) +# --------------------------------------------------------------------------- + + +class _PQTracer(_fx.Tracer): + """Tracer that treats all PQuant layer types (and standard torch.nn leaves) as atomic.""" + + _LEAF_TYPES = ( + PQDense, + PQConv2d, + PQConv1d, + PQBatchNorm1d, + PQBatchNorm2d, + PQLayerNorm, + PQAvgPool1d, + PQAvgPool2d, + PQMultiheadAttention, + ) + + def is_leaf_module(self, m: nn.Module, qualname: str) -> bool: + return isinstance(m, self._LEAF_TYPES) or super().is_leaf_module(m, qualname) + + +def convert_to_onnx_fx( + model: nn.Module, + input_shape: tuple, + output_path: str = "model.onnx", + opset: int = 13, + use_qonnx: bool = False, + store_integer_weights: bool = False, + integer_ops: bool = False, + include_clip: bool = True, +) -> onnx.ModelProto: + """ + Convert any PQuant nn.Module to ONNX using torch.fx symbolic tracing. + + Unlike convert_to_onnx(), this function works with arbitrary model topologies + including residual/skip connections, branches, and concatenations. It requires + the model to be symbolically traceable (no data-dependent control flow). + + Args match convert_to_onnx() exactly; see that function for parameter docs. + """ + model.eval() + quant_fn = _quant_node if use_qonnx else functools.partial(_qdq_node, include_clip=include_clip) + + graph = _PQTracer().trace(model) + gm = _fx.GraphModule(model, graph) + + # ShapeProp populates node.meta["tensor_meta"], which transpose/permute + # need to expand torch's two-arg .transpose(d0, d1) into a full ONNX perm. + from torch.fx.passes.shape_prop import ShapeProp + + with torch.no_grad(): + ShapeProp(gm).propagate(torch.zeros(1, *input_shape)) + + onnx_nodes: list[onnx.NodeProto] = [] + initializers: list[onnx.TensorProto] = [] + node_to_name: dict[_fx.Node, str] = {} + output_name: str = "" + + def _res(arg) -> str: + if isinstance(arg, _fx.Node): + return node_to_name[arg] + raise TypeError(f"Expected fx.Node, got {type(arg)}") + + def _binop_inputs(node: _fx.Node) -> list[str]: + # Like _res for both args, but lifts scalar literals (int/float/bool) + # to float32 initializers so patterns like ``x / 2.0`` work. + names: list[str] = [] + for i, a in enumerate(node.args[:2]): + if isinstance(a, _fx.Node): + names.append(node_to_name[a]) + elif isinstance(a, (int, float, bool)): + cname = f"{node.name}_arg{i}_const" + initializers.append(onh.from_array(np.array(float(a), dtype=np.float32), name=cname)) + names.append(cname) + else: + raise TypeError(f"FX export: unsupported binary-op arg type {type(a).__name__}") + return names + + def _rank(n: _fx.Node) -> int: + meta = n.meta.get("tensor_meta") + if meta is None or not hasattr(meta, "shape"): + raise RuntimeError(f"FX export: ShapeProp did not produce tensor_meta for {n.name!r}") + return len(meta.shape) + + def _swap_perm(rank: int, d0: int, d1: int) -> list[int]: + perm = list(range(rank)) + a, b = d0 % rank, d1 % rank + perm[a], perm[b] = perm[b], perm[a] + return perm + + def _resolve_perm_dims(args, rank: int) -> list[int]: + # Accept both permute(d0, d1, ...) and permute([d0, d1, ...]) shapes. + if len(args) == 1 and isinstance(args[0], (list, tuple)): + dims = args[0] + else: + dims = args + return [int(d) % rank for d in dims] + + for node in gm.graph.nodes: + if node.op == "placeholder": + node_to_name[node] = "input" + + elif node.op == "get_attr": + # Constant tensor attributes β€” store as initializer on first use. + # Retrieve the actual tensor from the GraphModule. + obj = gm + for part in node.target.split("."): + obj = getattr(obj, part) + attr_name = node.name + if isinstance(obj, torch.Tensor): + initializers.append(onh.from_array(obj.detach().cpu().numpy(), name=attr_name)) + node_to_name[node] = attr_name + + elif node.op == "call_module": + mod = gm.get_submodule(node.target) + mod_prefix = node.name.replace(".", "_") + if isinstance(mod, PQMultiheadAttention): + # node.args = (query, key, value[, key_padding_mask, attn_mask, ...]) + q_name = node_to_name[node.args[0]] + k_name = node_to_name[node.args[1]] if len(node.args) > 1 else q_name + v_name = node_to_name[node.args[2]] if len(node.args) > 2 else q_name + out_name, avg_attn_name = _add_mha( + mod, + mod_prefix, + q_name, + k_name, + v_name, + onnx_nodes, + initializers, + quant_fn, + use_qonnx, + store_integer_weights, + ) + # Store tuple so operator.getitem(node, 0/1) resolves correctly. + node_to_name[node] = (out_name, avg_attn_name) + else: + current = _emit_module( + mod, + mod_prefix, + node_to_name[node.args[0]], + onnx_nodes, + initializers, + quant_fn, + use_qonnx, + store_integer_weights, + integer_ops, + ) + node_to_name[node] = current + + elif node.op == "call_function": + fn = node.target + + if fn is _operator.getitem: + # Unpack a tuple output (e.g. from PQMultiheadAttention). + container = node_to_name[node.args[0]] + if not isinstance(container, tuple): + raise TypeError( + f"operator.getitem on non-tuple node {node.args[0].name!r} " f"is not supported in FX ONNX export" + ) + node_to_name[node] = container[node.args[1]] + continue + + if fn in (torch.add, _operator.add, _operator.iadd): + out = f"{node.name}_add" + onnx_nodes.append(oh.make_node("Add", inputs=_binop_inputs(node), outputs=[out])) + node_to_name[node] = out + + elif fn in (torch.mul, _operator.mul): + out = f"{node.name}_mul" + onnx_nodes.append(oh.make_node("Mul", inputs=_binop_inputs(node), outputs=[out])) + node_to_name[node] = out + + elif fn in (torch.sub, _operator.sub, _operator.isub): + out = f"{node.name}_sub" + onnx_nodes.append(oh.make_node("Sub", inputs=_binop_inputs(node), outputs=[out])) + node_to_name[node] = out + + elif fn in (torch.div, _operator.truediv, _operator.itruediv): + out = f"{node.name}_div" + onnx_nodes.append(oh.make_node("Div", inputs=_binop_inputs(node), outputs=[out])) + node_to_name[node] = out + + elif fn in (torch.matmul, _operator.matmul): + out = f"{node.name}_matmul" + onnx_nodes.append(oh.make_node("MatMul", inputs=_binop_inputs(node), outputs=[out])) + node_to_name[node] = out + + elif fn is torch.transpose: + # torch.transpose(t, d0, d1) swaps two dims; ONNX needs a full perm. + rank = _rank(node.args[0]) + perm = _swap_perm(rank, int(node.args[1]), int(node.args[2])) + out = f"{node.name}_transpose" + onnx_nodes.append(oh.make_node("Transpose", inputs=[_res(node.args[0])], outputs=[out], perm=perm)) + node_to_name[node] = out + + elif fn is torch.permute: + rank = _rank(node.args[0]) + perm = _resolve_perm_dims(node.args[1:], rank) + out = f"{node.name}_permute" + onnx_nodes.append(oh.make_node("Transpose", inputs=[_res(node.args[0])], outputs=[out], perm=perm)) + node_to_name[node] = out + + elif fn is torch.cat: + tensors = [_res(a) for a in node.args[0]] + dim = node.args[1] if len(node.args) > 1 else node.kwargs.get("dim", 0) + out = f"{node.name}_concat" + onnx_nodes.append(oh.make_node("Concat", inputs=tensors, outputs=[out], axis=int(dim))) + node_to_name[node] = out + + elif fn in (_F.relu, torch.relu): + out = f"{node.name}_relu" + onnx_nodes.append(oh.make_node("Relu", inputs=[_res(node.args[0])], outputs=[out])) + node_to_name[node] = out + + elif fn is torch.flatten: + start_dim = node.args[1] if len(node.args) > 1 else node.kwargs.get("start_dim", 0) + out = f"{node.name}_flatten" + onnx_nodes.append(oh.make_node("Flatten", inputs=[_res(node.args[0])], outputs=[out], axis=int(start_dim))) + node_to_name[node] = out + + else: + raise TypeError(f"Unsupported call_function for FX ONNX export: {fn}") + + elif node.op == "call_method": + x = _res(node.args[0]) + + if node.target == "relu": + out = f"{node.name}_relu" + onnx_nodes.append(oh.make_node("Relu", inputs=[x], outputs=[out])) + node_to_name[node] = out + + elif node.target == "flatten": + start_dim = node.args[1] if len(node.args) > 1 else node.kwargs.get("start_dim", 1) + out = f"{node.name}_flatten" + onnx_nodes.append(oh.make_node("Flatten", inputs=[x], outputs=[out], axis=int(start_dim))) + node_to_name[node] = out + + elif node.target in ("view", "reshape"): + shape_vals = [] + for a in node.args[1:]: + if not isinstance(a, int): + raise TypeError("Dynamic reshape (non-constant shape) is not supported in FX ONNX export") + shape_vals.append(a) + shape_name = f"{node.name}_shape" + out = f"{node.name}_reshape" + initializers.append(onh.from_array(np.array(shape_vals, dtype=np.int64), name=shape_name)) + onnx_nodes.append(oh.make_node("Reshape", inputs=[x, shape_name], outputs=[out])) + node_to_name[node] = out + + elif node.target == "transpose": + rank = _rank(node.args[0]) + perm = _swap_perm(rank, int(node.args[1]), int(node.args[2])) + out = f"{node.name}_transpose" + onnx_nodes.append(oh.make_node("Transpose", inputs=[x], outputs=[out], perm=perm)) + node_to_name[node] = out + + elif node.target == "permute": + rank = _rank(node.args[0]) + perm = _resolve_perm_dims(node.args[1:], rank) + out = f"{node.name}_permute" + onnx_nodes.append(oh.make_node("Transpose", inputs=[x], outputs=[out], perm=perm)) + node_to_name[node] = out + + elif node.target == "matmul": + out = f"{node.name}_matmul" + onnx_nodes.append(oh.make_node("MatMul", inputs=[x, _res(node.args[1])], outputs=[out])) + node_to_name[node] = out + + else: + raise TypeError(f"Unsupported call_method for FX ONNX export: {node.target!r}") + + elif node.op == "output": + ret = node.args[0] + if isinstance(ret, _fx.Node): + val = node_to_name[ret] + # MHA nodes store a tuple (out, avg_attn); expose the attention output. + output_name = val[0] if isinstance(val, tuple) else val + elif isinstance(ret, (tuple, list)) and len(ret) == 1: + val = node_to_name[ret[0]] + output_name = val[0] if isinstance(val, tuple) else val + else: + raise TypeError("Only single-output models are supported for FX ONNX export") + + with torch.no_grad(): + dummy_out = model(torch.zeros(1, *input_shape)) + output_shape = [None] + list(dummy_out.shape[1:]) + + batch_dim = oh.make_tensor_value_info("input", TensorProto.FLOAT, [None, *input_shape]) + output_vi = oh.make_tensor_value_info(output_name, TensorProto.FLOAT, output_shape) + + onnx_graph = oh.make_graph( + nodes=onnx_nodes, + name="pquant_onnx_fx", + inputs=[batch_dim], + outputs=[output_vi], + initializer=initializers, + ) + + opset_imports = [oh.make_opsetid("", opset)] + if use_qonnx: + opset_imports.append(oh.make_opsetid("qonnx.custom_op.general", 1)) + model_proto = oh.make_model(onnx_graph, opset_imports=opset_imports) + model_proto.ir_version = 6 + + onnx.checker.check_model(model_proto) + onnx.save(model_proto, output_path) + fmt = "QONNX" if use_qonnx else "ONNX (QDQ)" + logging.info("Saved %s model (FX) β†’ %s", fmt, output_path) + return model_proto + + +# --------------------------------------------------------------------------- +# usage example +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + import onnxruntime as ort + + import pquant + + cfg = pquant.cs_config() + cfg.quantization_parameters.granularity = "per-channel" + + model = nn.Sequential( + PQConv2d(cfg, in_channels=3, out_channels=16, kernel_size=3, padding=1), + PQBatchNorm2d(cfg, num_features=16), + nn.ReLU(), + PQAvgPool2d(cfg, kernel_size=2, stride=2), + nn.Flatten(), + PQDense(cfg, in_features=16 * 16 * 16, out_features=64), + nn.ReLU(), + PQDense(cfg, in_features=64, out_features=10), + ) + + x = torch.randn(4, 3, 32, 32) + with torch.no_grad(): + model(x) + + for module in model.modules(): + if hasattr(module, "apply_final_compression"): + module.apply_final_compression() + + model.eval() + with torch.no_grad(): + torch_out = model(x).numpy() + + qonnx_path = "model_qonnx.onnx" + onnx_path = "model_qdq.onnx" + convert_to_onnx(model, input_shape=(3, 32, 32), output_path=qonnx_path, use_qonnx=True) + convert_to_onnx(model, input_shape=(3, 32, 32), output_path=onnx_path, use_qonnx=False) + + from qonnx.core.modelwrapper import ModelWrapper + from qonnx.core.onnx_exec import execute_onnx + from qonnx.transformation.infer_shapes import InferShapes + + qmodel = ModelWrapper(qonnx_path) + qmodel.graph.input[0].type.tensor_type.shape.dim[0].dim_value = x.shape[0] + qmodel = qmodel.transform(InferShapes()) + input_name = qmodel.graph.input[0].name + output_name = qmodel.graph.output[0].name + qonnx_out = execute_onnx(qmodel, {input_name: x.numpy()})[output_name] + + sess = ort.InferenceSession(onnx_path) + onnx_out = sess.run(None, {sess.get_inputs()[0].name: x.numpy()})[0] + + print(f"\n{'':=<55}") # noqa: T201 + print(f" max |torch - qonnx| : {np.abs(torch_out - qonnx_out).max():.6f}") # noqa: T201 + print(f" max |torch - onnx| : {np.abs(torch_out - onnx_out).max():.6f}") # noqa: T201 + print(f" max |qonnx - onnx| : {np.abs(qonnx_out - onnx_out).max():.6f}") # noqa: T201 + print(f"{'':=<55}") # noqa: T201 diff --git a/src/pquant/core/torch/distillers.py b/src/pquant/core/torch/distillers.py new file mode 100644 index 0000000..fce5fe0 --- /dev/null +++ b/src/pquant/core/torch/distillers.py @@ -0,0 +1,686 @@ +from __future__ import annotations + +import os +import tempfile +from typing import Callable, Iterable + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.data + +from pquant.core.torch.layers import ( + PQWeightBiasBase, + get_model_losses, + post_epoch_functions, + post_pretrain_functions, + post_round_functions, + pre_epoch_functions, + pre_finetune_functions, +) + + +def get_module(model: nn.Module, name: str) -> nn.Module: + for n, m in model.named_modules(): + if n == name: + return m + raise ValueError(f"Module '{name}' not found in model") + + +def pq_layer_names(model: nn.Module) -> list[str]: + """Return names of all PQWeightBiasBase submodules in forward order.""" + return [name for name, m in model.named_modules() if isinstance(m, PQWeightBiasBase)] + + +class CachedDataset(torch.utils.data.Dataset): + """Per-batch cache backed by files in a temporary directory. + + Each file stores one ``(input, output)`` batch as a tuple. + Since each file has one batch, uses batch_size of 1 here. + """ + + def __init__(self, cache_dir: str, n_batches: int) -> None: + self.cache_dir = cache_dir + self.n_batches = n_batches + + def __len__(self) -> int: + return self.n_batches + + def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]: + return torch.load( + os.path.join(self.cache_dir, f"{idx:08d}.pt"), + weights_only=True, + ) + + +class LayerwiseDistiller: + """ + Distills a teacher model into a student model one PQ layer at a time. + + For each layer, all student parameters are frozen except that layer's, + and the layer is trained to match the teacher's outputs at that point. + + Args: + teacher: Reference model. Will be set to eval() and + gradients disabled during distillation. + student: PQuantML model. + loss_fn: Loss function between student and teacher activations. + precompute_layer_inputs: If True, run one pass over the dataloader at + the start of ``distill_layer``, capturing the teacher + layer's input and output. Pairs + are saved to a temporary directory on disk. All epoch loops + then call ``student_layer(layer_input)`` directly, skipping the full model forward entirely. + prefetch_workers: Number of DataLoader worker processes used to + prefetch cached batches from disk when + ``precompute_layer_inputs=True``. + cache_dir: Directory under which temporary per-layer cache + subdirectories are created when ``precompute_layer_inputs=True``. + """ + + def __init__( + self, + teacher: nn.Module, + student: nn.Module, + device: torch.device | None, + loss_fn: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] | None = None, + precompute_layer_inputs: bool = True, + prefetch_workers: int = 2, + cache_dir: str | None = None, + ): + self.teacher = teacher + self.student = student + self.device = device + self.loss_fn = loss_fn or F.mse_loss + self.precompute_layer_inputs = precompute_layer_inputs + self.prefetch_workers = prefetch_workers + self.cache_dir = cache_dir + + self.teacher.eval() + for param in self.teacher.parameters(): + param.requires_grad_(False) + + def precompute_layer_io( + self, + teacher_layer: nn.Module, + dataloader: Iterable, + cache_dir: str, + ) -> torch.utils.data.DataLoader: + """Cache ``(layer_input, layer_output)``. + + Runs one teacher pass and one frozen student pass per batch, saving + each pair. Returns a DataLoader which uses this saved data. + """ + teacher_captured_data: dict[str, torch.Tensor] = {} + + def teacher_post(m: nn.Module, inp: tuple, out: torch.Tensor) -> None: + teacher_output = out[0] if isinstance(out, tuple) else out + teacher_captured_data['out'] = teacher_output.detach().cpu() + + def teacher_pre(m: nn.Module, inp: tuple) -> None: + teacher_captured_data['inp'] = inp[0].detach().cpu() + + teacher_output_hook = teacher_layer.register_forward_hook(teacher_post) + teacher_input_hook = teacher_layer.register_forward_pre_hook(teacher_pre) + + n_batches = 0 + try: + with torch.no_grad(): + for x, _ in dataloader: + if self.device is not None: + x = x.to(self.device) + self.teacher(x) + torch.save( + (teacher_captured_data['inp'], teacher_captured_data['out']), + os.path.join(cache_dir, f"{n_batches:08d}.pt"), + ) + n_batches += 1 + finally: + teacher_output_hook.remove() + teacher_input_hook.remove() + + dataset = CachedDataset(cache_dir, n_batches) + return torch.utils.data.DataLoader( + dataset, + batch_size=1, + shuffle=True, + num_workers=self.prefetch_workers, + collate_fn=lambda b: b[0], + ) + + def val_layer_loss( + self, + teacher_layer: nn.Module, + student_layer: nn.Module, + val_dataloader: Iterable, + ) -> float: + """Compute mean validation loss for a single layer (no grad, eval mode). + + Always runs a full model forward pass with hooks β€” the val dataset is + never cached to disk, avoiding large intermediate feature map storage. + """ + t_captured: dict[str, torch.Tensor] = {} + s_captured: dict[str, torch.Tensor] = {} + + def teacher_post(m: nn.Module, inp: tuple, out) -> None: + t_captured['out'] = (out[0] if isinstance(out, tuple) else out).detach() + + def student_hook(m: nn.Module, inp: tuple, out) -> None: + s_captured['out'] = out[0] if isinstance(out, tuple) else out + + h_t = teacher_layer.register_forward_hook(teacher_post) + h_s = student_layer.register_forward_hook(student_hook) + + self.student.eval() + batch_losses: list[float] = [] + try: + with torch.no_grad(): + for x, _ in val_dataloader: + if self.device is not None: + x = x.to(self.device) + self.teacher(x) + self.student(x) + batch_losses.append(self.loss_fn(s_captured['out'], t_captured['out']).item()) + finally: + h_t.remove() + h_s.remove() + self.student.train() + + return sum(batch_losses) / len(batch_losses) + + def distill_layer( + self, + teacher_layer_name: str, + student_layer_name: str, + dataloader: Iterable, + optimizer_factory: Callable[[Iterable], torch.optim.Optimizer], + n_epochs: int, + val_dataloader: Iterable | None = None, + epoch_callback: Callable[[int, float, float | None, str], None] | None = None, + ) -> list[float]: + """ + Distill a single layer. + + Args: + teacher_layer_name: Named path to the module in the teacher, e.g. ``"layers.0"``. + student_layer_name: Named path to the corresponding module in the student. + dataloader: Iterable of ``(inputs, targets)`` batches. + optimizer_factory: Callable that accepts an iterable of parameters and + returns a fresh optimizer. Called once per layer with only + the target layer's parameters, so state never carries over + between layers. Example: + ``lambda params: torch.optim.Adam(params, lr=1e-4)``. + n_epochs: Number of full passes through the dataloader. + ``post_epoch_functions`` is called on the layer after + each pass. + val_dataloader: Optional validation dataloader. When provided, a + no-grad eval pass is run after every training epoch and + the resulting mean loss is passed to ``epoch_callback``. + epoch_callback: Called after each epoch with + ``(epoch, train_loss, val_loss, student_layer_name)`` + where ``val_loss`` is ``None`` when no ``val_dataloader`` + is given. Use this to drive per-epoch side effects such + as calling ``student.increment_alpha()``, stepping a + scheduler, or logging metrics. + + Returns: + List of per-epoch mean training losses. + """ + teacher_layer = get_module(self.teacher, teacher_layer_name) + student_layer = get_module(self.student, student_layer_name) + + # Freeze everything except the target layer (skip non-float params + # like bool pruning masks which cannot require gradients) + frozen: dict[str, bool] = {} + for name, param in self.student.named_parameters(): + if not param.is_floating_point(): + continue + frozen[name] = param.requires_grad + param.requires_grad_(name.startswith(student_layer_name)) + + optimizer = optimizer_factory(student_layer.parameters()) + + tmpdir = None + hooks = [] + epoch_losses: list[float] = [] + self.student.train() + + try: + if self.precompute_layer_inputs: + tmpdir = tempfile.TemporaryDirectory(prefix="ldistil_", dir=self.cache_dir or os.getcwd()) + dataloader = self.precompute_layer_io(teacher_layer, dataloader, self.device, tmpdir.name) + else: + teacher_out: dict[str, torch.Tensor] = {} + student_out: dict[str, torch.Tensor] = {} + + def _teacher_hook(m: nn.Module, inp: tuple, out) -> None: + t = out[0] if isinstance(out, tuple) else out + teacher_out["out"] = t.detach() + + def student_hook(m: nn.Module, inp: tuple, out) -> None: + t = out[0] if isinstance(out, tuple) else out + student_out["out"] = t + + h_t = teacher_layer.register_forward_hook(_teacher_hook) + h_s = student_layer.register_forward_hook(student_hook) + hooks = [h_t, h_s] + + for epoch in range(n_epochs): + if getattr(student_layer, 'enable_pruning', False): + student_layer.pruning_layer.pre_epoch_function(epoch, n_epochs) + batch_losses: list[float] = [] + + if self.precompute_layer_inputs: + for t_in, t_out in dataloader: + if self.device is not None: + t_in, t_out = t_in.to(self.device), t_out.to(self.device) + raw = student_layer(t_in) + s_out = raw[0] if isinstance(raw, tuple) else raw + loss = self.loss_fn(s_out, t_out) + optimizer.zero_grad() + loss.backward() + optimizer.step() + batch_losses.append(loss.item()) + else: + for x, _ in dataloader: + if self.device is not None: + x = x.to(self.device) + with torch.no_grad(): + self.teacher(x) + self.student(x) + t_out, s_out = teacher_out["out"], student_out["out"] + loss = self.loss_fn(s_out, t_out) + optimizer.zero_grad() + loss.backward() + optimizer.step() + batch_losses.append(loss.item()) + + mean_loss = sum(batch_losses) / len(batch_losses) + epoch_losses.append(mean_loss) + if getattr(student_layer, 'enable_pruning', False): + student_layer.pruning_layer.post_epoch_function(epoch, n_epochs) + val_loss: float | None = None + if val_dataloader is not None: + val_loss = self.val_layer_loss(teacher_layer, student_layer, val_dataloader) + if epoch_callback is not None: + epoch_callback(epoch, mean_loss, val_loss, student_layer_name) + finally: + for h in hooks: + h.remove() + if tmpdir is not None: + tmpdir.cleanup() + for name, param in self.student.named_parameters(): + if name in frozen: + param.requires_grad_(frozen[name]) + + return epoch_losses + + def distill_all( + self, + dataloader: Iterable, + optimizer_factory: Callable[[Iterable], torch.optim.Optimizer], + n_epochs: int, + layer_names: list[tuple[str, str]] | None = None, + val_dataloader: Iterable | None = None, + epoch_callback: Callable[[int, float, float | None, str], None] | None = None, + layer_callback: Callable[[str, str, list[float]], None] | None = None, + ) -> dict[tuple[str, str], list[float]]: + """ + Distill all PQ layers sequentially front-to-back. + + Args: + dataloader: Iterable of ``(inputs, targets)`` batches. + optimizer_factory: Callable ``(params) -> Optimizer``. Called fresh + for each layer so state never carries over. + n_epochs: Number of epochs (full dataloader passes) per layer. + layer_names: List of ``(teacher_layer_name, student_layer_name)`` + tuples to distill. Defaults to pairing each + ``PQWeightBiasBase`` submodule name with itself. + val_dataloader: Optional validation dataloader passed through to + each ``distill_layer`` call. + epoch_callback: Passed through to ``distill_layer``. Called after + each epoch with + ``(epoch, train_loss, val_loss, layer_name)`` + where ``val_loss`` is ``None`` when no + ``val_dataloader`` is given. + layer_callback: Called after each layer completes with + ``(teacher_layer_name, student_layer_name, losses)``. + + Returns: + Dict mapping ``(teacher_layer_name, student_layer_name)`` to list + of per-epoch mean losses. + """ + if layer_names is not None: + pairs = layer_names + else: + names = pq_layer_names(self.student) + pairs = [(n, n) for n in names] + history: dict[tuple[str, str], list[float]] = {} + + for t_name, s_name in pairs: + losses = self.distill_layer( + t_name, + s_name, + dataloader, + optimizer_factory, + n_epochs, + val_dataloader=val_dataloader, + epoch_callback=epoch_callback, + ) + history[(t_name, s_name)] = losses + if layer_callback is not None: + layer_callback(t_name, s_name, losses) + + return history + + +class ModelDistiller: + """ + Knowledge distillation at the model output level. + + The teacher's logits are used as soft targets for the student via KL + divergence loss. Because the teacher (16 raw classes) and student (10 + merged classes) have different output sizes, a raw-to-merged mapping + tensor is used to collapse the teacher logits to 10 classes before + computing the loss. + + Args: + teacher: Full-precision reference model (16-class output). Set to + eval() with gradients disabled. + student: Student model (10-class output). + teacher_transform: Optional callable applied to teacher logits before + the distillation loss is computed. Use this to map + teacher outputs to the student's output space, e.g. to + collapse classes. If ``None``, teacher logits are used + as-is (teacher and student must then have matching output + shapes). + loss_fn: Which distillation loss to use. One of: + ``"kl_ce"`` (default) β€” ``alpha * KL(student || teacher) + + (1 - alpha) * CE(student, labels)`` with temperature + scaling. Classic Hinton et al. formulation. + ``"kl"`` β€” KL divergence only, no hard label term. + Useful when ground-truth labels are unavailable or noisy. + ``"mse"`` β€” mean squared error directly on the logits, + no temperature scaling. Simpler baseline; equivalent to + KL under a uniform teacher distribution. + temperature: Softmax temperature for soft targets (default 4.0). + Only used by ``"kl_ce"`` and ``"kl"``. + alpha: Weight of the KL distillation loss (default 0.7). + The remaining ``(1 - alpha)`` weight is given to the + hard cross-entropy loss. Only used by ``"kl_ce"``. + precompute_teacher_outputs: If True (default), run a single inference + pass of the teacher over the entire dataset before + distillation begins, caching transformed teacher logits + to disk. All epoch loops then + use this cached loader and never call the teacher again. + Set to False only when the dataloader uses stochastic + augmentations that must remain live during training. + prefetch_workers: Number of DataLoader worker processes used to + prefetch cached batches from disk when + ``precompute_teacher_outputs=True``. Defaults to 2. + cache_dir: Directory under which the temporary cache is created + when ``precompute_teacher_outputs=True``. Defaults to + the current working directory. Avoid ``/tmp`` on Linux + as it is typically a RAM-backed ``tmpfs``. + """ + + LOSS_FN_OPTIONS = ("kl_ce", "kl", "mse") + + def __init__( + self, + teacher: nn.Module, + student: nn.Module, + pq_config, + device: torch.device | None, + teacher_transform: Callable[[torch.Tensor], torch.Tensor] | None = None, + loss_fn: str = "kl_ce", + temperature: float = 4.0, + alpha: float = 0.7, + precompute_teacher_outputs: bool = True, + prefetch_workers: int = 2, + cache_dir: str | None = None, + ): + if loss_fn not in self.LOSS_FN_OPTIONS: + raise ValueError(f"loss_fn must be one of {self.LOSS_FN_OPTIONS}, got '{loss_fn}'") + + self.teacher = teacher + self.student = student + self.pq_config = pq_config + self.teacher_transform = teacher_transform + self.loss_fn = loss_fn + self.T = temperature + self.alpha = alpha + self._precompute_teacher_outputs = precompute_teacher_outputs + self.prefetch_workers = prefetch_workers + self.cache_dir = cache_dir + self.device = device + self.teacher.eval() + for p in self.teacher.parameters(): + p.requires_grad_(False) + + def precompute_teacher_outputs( + self, + dataloader: Iterable, + cache_dir: str, + shuffle: bool = True, + ) -> torch.utils.data.DataLoader: + """Run one teacher inference pass and cache ``(x, teacher_output, y)`` to disk.""" + n_batches = 0 + self.teacher.eval() + with torch.no_grad(): + for x, y in dataloader: + if self.device is not None: + x = x.to(self.device) + teacher_logits = self.teacher(x) + teacher_out = self.teacher_transform(teacher_logits) if self.teacher_transform else teacher_logits + torch.save( + (x.cpu(), teacher_out.cpu(), y.cpu()), + os.path.join(cache_dir, f"{n_batches:08d}.pt"), + ) + n_batches += 1 + + dataset = CachedDataset(cache_dir, n_batches) + return torch.utils.data.DataLoader( + dataset, + batch_size=1, + shuffle=shuffle, + num_workers=self.prefetch_workers, + collate_fn=lambda b: b[0], + ) + + def kl_divergence(self, student_logits: torch.Tensor, teacher_merged: torch.Tensor) -> torch.Tensor: + _, C, _, _ = student_logits.shape + flat_s = student_logits.permute(0, 2, 3, 1).reshape(-1, C) + flat_t = teacher_merged.permute(0, 2, 3, 1).reshape(-1, C) + log_p_s = F.log_softmax(flat_s / self.T, dim=1) + p_t = F.softmax(flat_t / self.T, dim=1) + return F.kl_div(log_p_s, p_t, reduction="batchmean") * (self.T**2) + + def loss_kl_ce( + self, + student_logits: torch.Tensor, + teacher_logits: torch.Tensor, + labels: torch.Tensor, + ) -> torch.Tensor: + """alpha * KL(student || teacher) + (1 - alpha) * CE(student, labels).""" + kl_loss = self.kl_divergence(student_logits, teacher_logits) + ce_loss = F.cross_entropy(student_logits, labels, ignore_index=-1) + return self.alpha * kl_loss + (1 - self.alpha) * ce_loss + + def loss_kl( + self, + student_logits: torch.Tensor, + teacher_logits: torch.Tensor, + ) -> torch.Tensor: + """KL divergence only, no hard label term.""" + return self.kl_divergence(student_logits, teacher_logits) + + def loss_mse( + self, + student_logits: torch.Tensor, + teacher_logits: torch.Tensor, + ) -> torch.Tensor: + """MSE directly on logits, no temperature scaling.""" + return F.mse_loss(student_logits, teacher_logits) + + def compute_loss( + self, + student_logits: torch.Tensor, + teacher_logits: torch.Tensor, + labels: torch.Tensor, + ) -> torch.Tensor: + if self.loss_fn == "kl_ce": + return self.loss_kl_ce(student_logits, teacher_logits, labels) + elif self.loss_fn == "kl": + return self.loss_kl(student_logits, teacher_logits) + else: # mse + return self.loss_mse(student_logits, teacher_logits) + + def run_val_epoch( + self, + val_dataloader: Iterable, + ) -> float: + """Compute mean validation loss over one pass of val_dataloader (no grad, eval mode). + + When ``precompute_teacher_outputs=True`` the dataloader yields + ``(x, teacher_logits, y)`` and the cached teacher output is used directly. + Otherwise it yields ``(x, y)`` and the teacher is run live. + """ + self.student.eval() + batch_losses: list[float] = [] + with torch.no_grad(): + for batch in val_dataloader: + if self._precompute_teacher_outputs: + x, teacher_logits, y = batch + if self.device is not None: + x, teacher_logits, y = x.to(self.device), teacher_logits.to(self.device), y.to(self.device) + else: + x, y = batch + if self.device is not None: + x, y = x.to(self.device), y.to(self.device) + teacher_logits = self.teacher(x) + teacher_logits = self.teacher_transform(teacher_logits) if self.teacher_transform else teacher_logits + student_logits = self.student(x) + loss = self.compute_loss(student_logits, teacher_logits, y) + batch_losses.append(loss.item()) + self.student.train() + return sum(batch_losses) / len(batch_losses) + + def run_epoch( + self, + dataloader: Iterable, + optimizer: torch.optim.Optimizer, + ) -> float: + self.student.train() + batch_losses: list[float] = [] + + for batch in dataloader: + if self._precompute_teacher_outputs: + x, teacher_logits, y = batch + if self.device is not None: + x, teacher_logits, y = x.to(self.device), teacher_logits.to(self.device), y.to(self.device) + else: + x, y = batch + if self.device is not None: + x, y = x.to(self.device), y.to(self.device) + with torch.no_grad(): + teacher_logits = self.teacher(x) + teacher_logits = self.teacher_transform(teacher_logits) if self.teacher_transform else teacher_logits + + student_logits = self.student(x) + loss = self.compute_loss(student_logits, teacher_logits, y) + loss = get_model_losses(self.student, loss) + + optimizer.zero_grad() + loss.backward() + optimizer.step() + batch_losses.append(loss.item()) + + return sum(batch_losses) / len(batch_losses) + + def distill( + self, + dataloader: Iterable, + optimizer: torch.optim.Optimizer, + val_dataloader: Iterable | None = None, + epoch_callback: Callable[[int, float, float | None], None] | None = None, + ) -> list[float]: + """ + Run model distillation following the PQuantML training pipeline. + + Args: + dataloader: Dataloader provided by user. + optimizer: Optimizer for student parameters. + val_dataloader: Optional validation dataloader. When provided, a + no-grad eval pass is run after every training epoch + and the resulting mean loss is passed to + ``epoch_callback``. + epoch_callback: Called after each epoch with + ``(epoch, train_loss, val_loss)`` where + ``val_loss`` is ``None`` when no ``val_dataloader`` + is given. + + Returns: + List of per-epoch mean training losses. + """ + training_parameters = self.pq_config.training_parameters + + tmpdir = None + tmpdir_val = None + epoch_losses: list[float] = [] + global_epoch = 0 + + try: + if self._precompute_teacher_outputs: + tmpdir = tempfile.TemporaryDirectory(prefix="mdistil_", dir=self.cache_dir or os.getcwd()) + dataloader = self.precompute_teacher_outputs(dataloader, tmpdir.name) + if val_dataloader is not None: + tmpdir_val = tempfile.TemporaryDirectory(prefix="mdistil_val_", dir=self.cache_dir or os.getcwd()) + val_dataloader = self.precompute_teacher_outputs(val_dataloader, tmpdir_val.name, shuffle=False) + + for e in range(training_parameters.pretraining_epochs): + pre_epoch_functions(self.student, e, training_parameters.pretraining_epochs) + mean_loss = self.run_epoch(dataloader, optimizer) + epoch_losses.append(mean_loss) + post_epoch_functions(self.student, e, training_parameters.pretraining_epochs) + val_loss: float | None = None + if val_dataloader is not None: + val_loss = self.run_val_epoch(val_dataloader) + if epoch_callback is not None: + epoch_callback(global_epoch, mean_loss, val_loss) + global_epoch += 1 + + post_pretrain_functions(self.student, self.pq_config, train_loader=dataloader) + + for _ in range(training_parameters.rounds): + for e in range(training_parameters.epochs): + pre_epoch_functions(self.student, e, training_parameters.epochs) + mean_loss = self.run_epoch(dataloader, optimizer) + epoch_losses.append(mean_loss) + post_epoch_functions(self.student, e, training_parameters.epochs) + val_loss = None + if val_dataloader is not None: + val_loss = self.run_val_epoch(val_dataloader) + if epoch_callback is not None: + epoch_callback(global_epoch, mean_loss, val_loss) + global_epoch += 1 + post_round_functions(self.student) + + pre_finetune_functions(self.student) + for e in range(training_parameters.fine_tuning_epochs): + pre_epoch_functions(self.student, e, training_parameters.fine_tuning_epochs) + mean_loss = self.run_epoch(dataloader, optimizer) + epoch_losses.append(mean_loss) + val_loss = None + if val_dataloader is not None: + val_loss = self.run_val_epoch(val_dataloader) + if epoch_callback is not None: + epoch_callback(global_epoch, mean_loss, val_loss) + post_epoch_functions(self.student, e, training_parameters.fine_tuning_epochs) + global_epoch += 1 + finally: + if tmpdir is not None: + tmpdir.cleanup() + if tmpdir_val is not None: + tmpdir_val.cleanup() + + return epoch_losses diff --git a/src/pquant/core/torch/layers.py b/src/pquant/core/torch/layers.py index 83235d1..1a28b6f 100644 --- a/src/pquant/core/torch/layers.py +++ b/src/pquant/core/torch/layers.py @@ -87,6 +87,7 @@ def __init__( self.post_fitcompress_calibration = False self.saved_inputs = [] self.saved_outputs = [] + self.config = config def check_is_built(self, input_shape): if self.built: @@ -103,6 +104,7 @@ def check_is_built(self, input_shape): is_data=True, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, ) self.weight_quantizer = Quantizer( k=torch.tensor(self.k_weight), @@ -139,6 +141,7 @@ def check_is_built(self, input_shape): is_data=True, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, ) self.n_parallel = ops.prod(tuple(input_shape)[1:-1]) @@ -663,6 +666,7 @@ def build(self, input_shape): is_data=True, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, ) self.output_quantizer = Quantizer( k=torch.tensor(self.k_output), @@ -674,6 +678,7 @@ def build(self, input_shape): is_data=True, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, ) self.input_shape = (1,) + input_shape[1:] @@ -872,6 +877,7 @@ def check_is_built(self, input_shape): is_data=True, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, ) self.weight_quantizer = Quantizer( k=torch.tensor(self.k_weight), @@ -1035,6 +1041,7 @@ def check_is_built(self, input_shape): is_data=True, hgq_gamma=self.hgq_gamma, place="datalane", + dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, ) self.weight_quantizer = Quantizer( k=torch.tensor(self.k_weight), @@ -1110,7 +1117,7 @@ def hgq_loss(self): loss += self.input_quantizer.hgq_loss() return loss - def post_pretrain_function(self): + def post_pre_train_function(self): self.is_pretraining = False def forward(self, input: torch.Tensor) -> torch.Tensor: @@ -1124,6 +1131,411 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: return super().forward(input) +class PQLayerNorm(nn.LayerNorm): + def __init__( + self, + config, + normalized_shape: Union[int, Tuple[int, ...], torch.Size], + eps: float = 1e-5, + elementwise_affine: bool = True, + bias: bool = True, + device=None, + dtype=None, + quantize_input=True, + quantize_output=False, + in_quant_bits: Tuple[T, T, T] = None, + out_quant_bits: Tuple[T, T, T] = None, + weight_quant_bits: Tuple[T, T, T] = None, + bias_quant_bits: Tuple[T, T, T] = None, + ): + try: + super().__init__(normalized_shape, eps, elementwise_affine, bias, device=device, dtype=dtype) + except TypeError: + # Older torch versions don't accept the bias kwarg + super().__init__(normalized_shape, eps, elementwise_affine, device=device, dtype=dtype) + if in_quant_bits is not None: + self.k_input, self.i_input, self.f_input = in_quant_bits + else: + self.k_input = config.quantization_parameters.default_data_keep_negatives + self.i_input = config.quantization_parameters.default_data_integer_bits + self.f_input = config.quantization_parameters.default_data_fractional_bits + + if out_quant_bits is not None: + self.k_output, self.i_output, self.f_output = out_quant_bits + else: + self.k_output = config.quantization_parameters.default_data_keep_negatives + self.i_output = config.quantization_parameters.default_data_integer_bits + self.f_output = config.quantization_parameters.default_data_fractional_bits + + if weight_quant_bits is not None: + self.k_weight, self.i_weight, self.f_weight = weight_quant_bits + else: + self.k_weight = config.quantization_parameters.default_weight_keep_negatives + self.i_weight = config.quantization_parameters.default_weight_integer_bits + self.f_weight = config.quantization_parameters.default_weight_fractional_bits + if bias_quant_bits is not None: + self.k_bias, self.i_bias, self.f_bias = bias_quant_bits + else: + self.k_bias = config.quantization_parameters.default_weight_keep_negatives + self.i_bias = config.quantization_parameters.default_weight_integer_bits + self.f_bias = config.quantization_parameters.default_weight_fractional_bits + self.overflow_mode_parameters = config.quantization_parameters.overflow_mode_parameters + self.overflow_mode_data = config.quantization_parameters.overflow_mode_data + self.round_mode = config.quantization_parameters.round_mode + self.use_hgq = config.quantization_parameters.use_high_granularity_quantization + self.hgq_gamma = config.quantization_parameters.hgq_gamma + self.hgq_beta = config.quantization_parameters.hgq_beta + self.enable_quantization = config.quantization_parameters.enable_quantization + self.use_fitcompress = config.fitcompress_parameters.enable_fitcompress + self.config = config + self.quantize_input = quantize_input + self.quantize_output = quantize_output + if self.weight is not None: + self._weight = nn.Parameter(self.weight.clone()).to(self.weight.device) + self.register_parameter("_weight", self._weight) + else: + self.register_parameter("_weight", None) + if self.bias is not None: + self._bias = nn.Parameter(self.bias.clone()).to(self.bias.device) + self.register_parameter("_bias", self._bias) + else: + self.register_parameter("_bias", None) + self.built = False + self.final_compression_done = False + self.is_pretraining = True + self.post_fitcompress_calibration = False + self.saved_inputs = [] + + def check_is_built(self, input_shape): + if self.built: + return + self.built = True + self.input_quantizer = Quantizer( + k=torch.tensor(self.k_input), + i=torch.tensor(self.i_input), + f=torch.tensor(self.f_input), + overflow=self.overflow_mode_data, + round_mode=self.round_mode, + is_heterogeneous=self.use_hgq, + is_data=True, + hgq_gamma=self.hgq_gamma, + place="datalane", + dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, + ) + self.output_quantizer = Quantizer( + k=torch.tensor(self.k_output), + i=torch.tensor(self.i_output), + f=torch.tensor(self.f_output), + overflow=self.overflow_mode_data, + round_mode=self.round_mode, + is_heterogeneous=self.use_hgq, + is_data=True, + hgq_gamma=self.hgq_gamma, + place="datalane", + dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, + ) + self.weight_quantizer = Quantizer( + k=torch.tensor(self.k_weight), + i=torch.tensor(self.i_weight), + f=torch.tensor(self.f_weight), + round_mode=self.round_mode, + overflow=self.overflow_mode_parameters, + is_data=False, + is_heterogeneous=self.use_hgq, + place="weight", + ) + self.bias_quantizer = Quantizer( + k=torch.tensor(self.k_bias), + i=torch.tensor(self.i_bias), + f=torch.tensor(self.f_bias), + round_mode=self.round_mode, + overflow=self.overflow_mode_parameters, + is_data=False, + is_heterogeneous=self.use_hgq, + place="bias", + ) + if self.use_hgq: + self.input_quantizer.quantizer.build(input_shape) + self.output_quantizer.quantizer.build(input_shape) + self.input_shape = (1,) + tuple(input_shape[1:]) + + def apply_final_compression(self): + self.final_compression_done = True + if self._weight is not None: + self._weight.data = self.weight + if self._bias is not None: + self._bias.data = self.bias + + def get_input_quantization_bits(self): + return self.input_quantizer.get_quantization_bits() + + def get_output_quantization_bits(self): + return self.output_quantizer.get_quantization_bits() + + def get_weight_quantization_bits(self): + return self.weight_quantizer.get_quantization_bits() + + def get_bias_quantization_bits(self): + return self.bias_quantizer.get_quantization_bits() + + def is_fitcompress_pretraining(self): + return self.is_pretraining and self.use_fitcompress + + @property + def weight(self): + if self._weight is None: + return None + if self.enable_quantization and not self.final_compression_done and not self.is_fitcompress_pretraining(): + return self.weight_quantizer(self._weight) + return self._weight + + @property + def bias(self): + if self._bias is None: + return None + if self.enable_quantization and not self.final_compression_done and not self.is_fitcompress_pretraining(): + return self.bias_quantizer(self._bias) + return self._bias + + def ebops(self): + return 0.0 + + def hgq_loss(self): + if self.is_pretraining or not self.use_hgq: + return ops.convert_to_tensor(0.0) + loss = self.hgq_beta * self.ebops() + if self._weight is not None: + loss += self.weight_quantizer.hgq_loss() + if self._bias is not None: + loss += self.bias_quantizer.hgq_loss() + if self.quantize_input: + loss += self.input_quantizer.hgq_loss() + if self.quantize_output: + loss += self.output_quantizer.hgq_loss() + return loss + + def post_pre_train_function(self): + self.is_pretraining = False + + def forward(self, input: torch.Tensor) -> torch.Tensor: + self.check_is_built(input.shape) + if self.quantize_input and self.enable_quantization: + if not self.is_fitcompress_pretraining(): + input = self.input_quantizer(input) + else: + if self.post_fitcompress_calibration: + self.saved_inputs.append(input) + out = F.layer_norm(input, self.normalized_shape, self.weight, self.bias, self.eps) + if self.quantize_output and self.enable_quantization and not self.is_fitcompress_pretraining(): + out = self.output_quantizer(out) + return out + + def extra_repr(self) -> str: + return ( + f"normalized_shape={tuple(self.normalized_shape)}, eps={self.eps}, " + f"elementwise_affine={self.elementwise_affine}, " + f"quantize_input={self.quantize_input}, quantize_output={self.quantize_output}" + ) + + +class PQMultiheadAttention(nn.Module): + """Multi-head attention with quantization support, implemented without F.multihead_attention. + + Uses separate PQDense projections for Q, K, V, and output, and computes + scaled dot-product attention manually. + + Args: + config: PQuant configuration object. + embed_dim: Total embedding dimension. + num_heads: Number of attention heads. + dropout: Dropout probability on attention weights. + bias: Whether to add bias to projection layers. + kdim: Key feature dimension (defaults to embed_dim). + vdim: Value feature dimension (defaults to embed_dim). + batch_first: If True, input/output tensors are (batch, seq, feature). + If False (default), tensors are (seq, batch, feature). + quantize_input: Whether to quantize Q/K/V projection inputs. + quantize_output: Whether to quantize projection outputs. + quantize_attn_weights: Whether to quantize attention weights after softmax. + in_quant_bits: (k, i, f) bits for input quantization. + weight_quant_bits: (k, i, f) bits for weight quantization. + bias_quant_bits: (k, i, f) bits for bias quantization. + out_quant_bits: (k, i, f) bits for output quantization. + attn_quant_bits: (k, i, f) bits for attention weight quantization. + """ + + def __init__( + self, + config, + embed_dim: int, + num_heads: int, + dropout: float = 0.0, + bias: bool = True, + kdim: int = None, + vdim: int = None, + batch_first: bool = False, + quantize_input: bool = True, + quantize_output: bool = False, + quantize_attn_weights: bool = False, + quantize_attn_scores: bool = False, + quantize_context: bool = False, + approximate_softmax: bool = False, + in_quant_bits: Tuple[T, T, T] = None, + weight_quant_bits: Tuple[T, T, T] = None, + bias_quant_bits: Tuple[T, T, T] = None, + out_quant_bits: Tuple[T, T, T] = None, + attn_quant_bits: Tuple[T, T, T] = None, + attn_score_quant_bits: Tuple[T, T, T] = None, + context_quant_bits: Tuple[T, T, T] = None, + **kwargs, + ): + super().__init__(**kwargs) + assert embed_dim % num_heads == 0, "embed_dim must be divisible by num_heads" + + self.embed_dim = embed_dim + self.num_heads = num_heads + self.head_dim = embed_dim // num_heads + self.batch_first = batch_first + self.dropout = dropout + self.quantize_attn_weights = quantize_attn_weights + self.quantize_attn_scores = quantize_attn_scores + self.quantize_context = quantize_context + self.approximate_softmax = approximate_softmax + self.scale = self.head_dim**-0.5 + self.softmax = nn.Softmax(dim=-1) + + kdim = kdim if kdim is not None else embed_dim + vdim = vdim if vdim is not None else embed_dim + + proj_kwargs = dict( + bias=bias, + quantize_input=quantize_input, + quantize_output=quantize_output, + in_quant_bits=in_quant_bits, + weight_quant_bits=weight_quant_bits, + bias_quant_bits=bias_quant_bits, + out_quant_bits=out_quant_bits, + ) + self.q_proj = PQDense(config, embed_dim, embed_dim, enable_pruning=False, **proj_kwargs) + self.k_proj = PQDense(config, kdim, embed_dim, enable_pruning=False, **proj_kwargs) + self.v_proj = PQDense(config, vdim, embed_dim, enable_pruning=False, **proj_kwargs) + self.out_proj = PQDense(config, embed_dim, embed_dim, **proj_kwargs) + + self.attn_dropout = None + + def _make_data_quantizer(bits): + if bits is not None: + k, i, f = bits + else: + k = config.quantization_parameters.default_data_keep_negatives + i = config.quantization_parameters.default_data_integer_bits + f = config.quantization_parameters.default_data_fractional_bits + return Quantizer( + k=torch.tensor(k), + i=torch.tensor(i), + f=torch.tensor(f), + overflow=config.quantization_parameters.overflow_mode_data, + round_mode=config.quantization_parameters.round_mode, + is_heterogeneous=config.quantization_parameters.use_high_granularity_quantization, + is_data=True, + hgq_gamma=config.quantization_parameters.hgq_gamma, + place="datalane", + dynamic_data=config.quantization_parameters.dynamic_data_quantization, + ) + + if quantize_attn_weights: + self.attn_weight_quantizer = _make_data_quantizer(attn_quant_bits) + if quantize_attn_scores: + self.attn_score_quantizer = _make_data_quantizer(attn_score_quant_bits) + if quantize_context: + self.context_quantizer = _make_data_quantizer(context_quant_bits) + self.enable_quantization = config.quantization_parameters.enable_quantization + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + key_padding_mask: Optional[torch.Tensor] = None, + attn_mask: Optional[torch.Tensor] = None, + need_weights: bool = True, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + if self.batch_first: + # (B, T, E) -> keep as-is + B, T, _ = query.shape + S = key.shape[1] + else: + # (T, B, E) -> (B, T, E) + query = query.transpose(0, 1) + key = key.transpose(0, 1) + value = value.transpose(0, 1) + B, T, _ = query.shape + S = key.shape[1] + + q = self.q_proj(query) # (B, T, E) + k = self.k_proj(key) # (B, S, E) + v = self.v_proj(value) # (B, S, E) + + # Reshape to (B, H, T/S, head_dim) + q = q.view(B, T, self.num_heads, self.head_dim).transpose(1, 2) + k = k.view(B, S, self.num_heads, self.head_dim).transpose(1, 2) + v = v.view(B, S, self.num_heads, self.head_dim).transpose(1, 2) + + # Scaled dot-product attention scores: (B, H, T, S) + attn_scores = torch.matmul(q, k.transpose(-2, -1)) * self.scale + + if attn_mask is not None: + if attn_mask.dim() == 2: + # (T, S) -> (1, 1, T, S), broadcast over batch and heads + attn_mask = attn_mask.unsqueeze(0).unsqueeze(0) + elif attn_mask.dim() == 3: + # (B*H, T, S) -> (B, H, T, S) + attn_mask = attn_mask.view(B, self.num_heads, T, S) + attn_scores = attn_scores + attn_mask + + if key_padding_mask is not None: + # key_padding_mask: (B, S), True means ignore + attn_scores = attn_scores.masked_fill(key_padding_mask.unsqueeze(1).unsqueeze(2), float("-inf")) + + if self.quantize_attn_scores and self.enable_quantization: + attn_scores = self.attn_score_quantizer(attn_scores) + + attn_weights = self.softmax(attn_scores) + + if self.quantize_attn_weights and self.enable_quantization: + attn_weights = self.attn_weight_quantizer(attn_weights) + + if self.attn_dropout is not None and self.training: + attn_weights = self.attn_dropout(attn_weights) + + # Weighted sum of values: (B, H, T, head_dim) + out = torch.matmul(attn_weights, v) + + if self.quantize_context and self.enable_quantization: + out = self.context_quantizer(out) + + # Merge heads: (B, T, E) + out = out.transpose(1, 2).contiguous().view(B, T, self.embed_dim) + out = self.out_proj(out) + + if not self.batch_first: + out = out.transpose(0, 1) + + if need_weights: + # Average attention weights over heads: (B, T, S) + return out, attn_weights.mean(dim=1) + return out, None + + def extra_repr(self) -> str: + return ( + f"embed_dim={self.embed_dim}, num_heads={self.num_heads}, " + f"dropout={self.dropout}, batch_first={self.batch_first}, " + f"quantize_attn_scores={self.quantize_attn_scores}, " + f"quantize_attn_weights={self.quantize_attn_weights}, " + f"quantize_context={self.quantize_context}" + ) + + def add_layer_specific_quantization_to_model(name, layer, config): if isinstance(layer, PQWeightBiasBase): if name in config.quantization_parameters.layer_specific: @@ -1192,6 +1604,41 @@ def add_layer_specific_quantization_to_model(name, layer, config): if "quantize" in layer_config["input"]: quantize = layer_config["input"]["quantize"] layer.quantize_input = quantize + elif layer.__class__ == PQLayerNorm: + if name in config.quantization_parameters.layer_specific: + layer_config = config.quantization_parameters.layer_specific[name] + if "weight" in layer_config: + if "keep_negatives" in layer_config["weight"]: + layer.k_weight = torch.tensor(layer_config["weight"]["keep_negatives"]) + if "integer_bits" in layer_config["weight"]: + layer.i_weight = torch.tensor(layer_config["weight"]["integer_bits"]) + if "fractional_bits" in layer_config["weight"]: + layer.f_weight = torch.tensor(layer_config["weight"]["fractional_bits"]) + if "bias" in layer_config: + if "keep_negatives" in layer_config["bias"]: + layer.k_bias = torch.tensor(layer_config["bias"]["keep_negatives"]) + if "integer_bits" in layer_config["bias"]: + layer.i_bias = torch.tensor(layer_config["bias"]["integer_bits"]) + if "fractional_bits" in layer_config["bias"]: + layer.f_bias = torch.tensor(layer_config["bias"]["fractional_bits"]) + if "input" in layer_config: + if "keep_negatives" in layer_config["input"]: + layer.k_input = torch.tensor(layer_config["input"]["keep_negatives"]) + if "integer_bits" in layer_config["input"]: + layer.i_input = torch.tensor(layer_config["input"]["integer_bits"]) + if "fractional_bits" in layer_config["input"]: + layer.f_input = torch.tensor(layer_config["input"]["fractional_bits"]) + if "quantize" in layer_config["input"]: + layer.quantize_input = layer_config["input"]["quantize"] + if "output" in layer_config: + if "keep_negatives" in layer_config["output"]: + layer.k_output = torch.tensor(layer_config["output"]["keep_negatives"]) + if "integer_bits" in layer_config["output"]: + layer.i_output = torch.tensor(layer_config["output"]["integer_bits"]) + if "fractional_bits" in layer_config["output"]: + layer.f_output = torch.tensor(layer_config["output"]["fractional_bits"]) + if "quantize" in layer_config["output"]: + layer.quantize_output = layer_config["output"]["quantize"] elif layer.__class__ in [PQAvgPool1d, PQAvgPool2d]: if name in config.quantization_parameters.layer_specific: layer_config = config.quantization_parameters.layer_specific[name] @@ -1329,6 +1776,22 @@ def add_quantized_activations_to_model_layer(module, config, prefix=""): ) new_layer = add_layer_specific_quantization_to_model(full_name, new_layer, config) setattr(module, name, new_layer) + elif layer.__class__ == nn.LayerNorm: + ln_kwargs = dict( + normalized_shape=layer.normalized_shape, + eps=layer.eps, + elementwise_affine=layer.elementwise_affine, + quantize_input=quantize_input, + quantize_output=quantize_output, + ) + new_layer = PQLayerNorm(config, **ln_kwargs) + if layer.elementwise_affine: + if layer.weight is not None and new_layer._weight is not None: + new_layer._weight.data.copy_(layer.weight.data) + if layer.bias is not None and new_layer._bias is not None: + new_layer._bias.data.copy_(layer.bias.data) + new_layer = add_layer_specific_quantization_to_model(full_name, new_layer, config) + setattr(module, name, new_layer) else: layer = add_quantized_activations_to_model_layer(layer, config, full_name) return module @@ -1443,7 +1906,7 @@ def add_pruning_to_model(module, config, prefix=""): def apply_final_compression(module): for layer in module.modules(): - if isinstance(layer, (PQWeightBiasBase, PQBatchNorm2d, PQBatchNorm1d, Quantizer)): + if isinstance(layer, (PQWeightBiasBase, PQBatchNorm2d, PQBatchNorm1d, PQLayerNorm, Quantizer)): layer.apply_final_compression() return module @@ -1503,7 +1966,18 @@ def post_pretrain_functions(model, config, train_loader=None, loss_function=None for layer in model.modules(): if isinstance( - layer, (PQConv2d, PQConv1d, PQDense, PQActivation, PQBatchNorm2d, PQBatchNorm1d, PQAvgPoolBase, Quantizer) + layer, + ( + PQConv2d, + PQConv1d, + PQDense, + PQActivation, + PQBatchNorm2d, + PQBatchNorm1d, + PQLayerNorm, + PQAvgPoolBase, + Quantizer, + ), ): # Trigger it here to enable quantization before FITCompress layer.post_pre_train_function() @@ -1523,7 +1997,18 @@ def post_pretrain_functions(model, config, train_loader=None, loss_function=None else: for layer in model.modules(): if isinstance( - layer, (PQConv2d, PQConv1d, PQDense, PQActivation, PQBatchNorm2d, PQBatchNorm1d, PQAvgPoolBase, Quantizer) + layer, + ( + PQConv2d, + PQConv1d, + PQDense, + PQActivation, + PQBatchNorm2d, + PQBatchNorm1d, + PQLayerNorm, + PQAvgPoolBase, + Quantizer, + ), ): layer.post_pre_train_function() if config.pruning_parameters.pruning_method == "pdp" or ( @@ -1591,7 +2076,7 @@ def get_model_losses(model, losses): if layer.use_hgq: loss += layer.hgq_loss() losses += loss - elif isinstance(layer, (PQAvgPool1d, PQAvgPool2d, PQBatchNorm2d, PQBatchNorm1d, PQActivation)): + elif isinstance(layer, (PQAvgPool1d, PQAvgPool2d, PQBatchNorm2d, PQBatchNorm1d, PQLayerNorm, PQActivation)): if layer.use_hgq: losses += layer.hgq_loss() return losses @@ -1742,7 +2227,7 @@ def get_ebops(model, **kwargs): for m in model.modules(): if isinstance(m, (PQWeightBiasBase)): ebops += m.ebops(include_mask=m.enable_pruning) - elif isinstance(m, (PQAvgPoolBase, PQBatchNorm1d, PQBatchNorm2d, PQActivation)): + elif isinstance(m, (PQAvgPoolBase, PQBatchNorm1d, PQBatchNorm2d, PQLayerNorm, PQActivation)): ebops += m.ebops() return ebops diff --git a/src/pquant/core/torch/quantizer.py b/src/pquant/core/torch/quantizer.py index 65e7998..1bf21cf 100644 --- a/src/pquant/core/torch/quantizer.py +++ b/src/pquant/core/torch/quantizer.py @@ -20,28 +20,27 @@ def __init__( granularity='per_tensor', hgq_gamma=0, place="datalane", + dynamic_data=True, ): super().__init__() - self.k = torch.nn.Parameter(torch.tensor(k), requires_grad=False) + self.k = torch.nn.Parameter(torch.tensor(float(k)), requires_grad=False) self.overflow = overflow self.b_init = k + i + f self.round_mode = round_mode self.use_hgq = is_heterogeneous self.is_data = is_data + self.dynamic_data = dynamic_data self.i_init = i self.f_init = f self.i = torch.nn.Parameter(torch.tensor(i), requires_grad=False) self.f = torch.nn.Parameter(torch.tensor(f), requires_grad=False) self.b = torch.nn.Parameter(torch.tensor(i + k + f), requires_grad=False) + self.granularity = granularity.value if isinstance(granularity, Enum) else granularity self.quantizer = create_quantizer( - self.k, i, f, self.overflow, self.round_mode, self.use_hgq, self.is_data, gamma=hgq_gamma + self.k, i, f, self.overflow, self.round_mode, self.use_hgq, self.is_data, hgq_gamma ) self.is_pretraining = True self.hgq_gamma = hgq_gamma - if isinstance(granularity, Enum): - self.granularity = granularity.value - else: - self.granularity = granularity self.final_compression_done = nn.Parameter(torch.tensor(False), requires_grad=False) if self.granularity == 'per_tensor': self.initialize_quantization_parameters(self.i_init, self.f_init) @@ -68,7 +67,24 @@ def set_quantization_bits(self, i, f): def post_pre_train_function(self): self.is_pretraining = False - def compute_dynamic_bits(self, x): + def calculate_bits_from_abs(self, abs_x): + m = torch.ceil(torch.log2(abs_x + 1e-6)) + int_bits = torch.clamp(m, min=0) + b = self.b if hasattr(self, "b") else self.k + self.i_init + self.f_init + frac_bits = torch.clamp(b - int_bits - self.k, min=0) + return int_bits, frac_bits + + def compute_data_dynamic_bits(self, x): + if not (self.training and self.dynamic_data): + _, i, f = self.get_quantization_bits() + return i, f + abs_x = torch.amax(torch.abs(x)) + return self.calculate_bits_from_abs(abs_x) + + def compute_weight_dynamic_bits(self, x): + if self.granularity == "per_tensor": + _, i, f = self.get_quantization_bits() + return i, f if self.granularity == "per_channel": if x.ndim == 2: abs_x = torch.amax(torch.abs(x), dim=1, keepdim=True) @@ -80,12 +96,12 @@ def compute_dynamic_bits(self, x): abs_x = torch.abs(x) else: raise ValueError("The selected granularity is not supported.") + return self.calculate_bits_from_abs(abs_x) - m = torch.ceil(torch.log2(abs_x + 1e-6)) - int_bits = torch.clamp(m, min=0) - b = self.b if hasattr(self, "b") else self.k + self.i_init + self.f_init - frac_bits = torch.clamp(b - int_bits - self.k, min=0) - return int_bits, frac_bits + def compute_dynamic_bits(self, x): + if self.is_data: + return self.compute_data_dynamic_bits(x) + return self.compute_weight_dynamic_bits(x) def forward(self, x): if self.use_hgq: @@ -94,12 +110,7 @@ def forward(self, x): self.initialize_quantization_parameters(i, f) return x else: - if self.granularity == 'per_tensor': - self.initialize_quantization_parameters(self.i_init, self.f_init) - _, i, f = self.get_quantization_bits() - return self.quantizer(x, k=self.k, i=i, f=f, training=self.training) - else: - i, f = self.compute_dynamic_bits(x) + i, f = self.compute_dynamic_bits(x) self.initialize_quantization_parameters(i, f) self.i.data = i self.f.data = f diff --git a/src/pquant/data_models/quantization_model.py b/src/pquant/data_models/quantization_model.py index a3c71b7..db8e2db 100644 --- a/src/pquant/data_models/quantization_model.py +++ b/src/pquant/data_models/quantization_model.py @@ -19,6 +19,7 @@ class BaseQuantizationModel(BaseModel): quantize_input: bool = Field(default=True) quantize_output: bool = Field(default=False) granularity: QuantizationGranularity = Field(default=QuantizationGranularity.PER_TENSOR) + dynamic_data_quantization: bool = Field(default=False) enable_quantization: bool = Field(default=True) hgq_gamma: float = Field(default=0.0003) hgq_beta: float = Field(default=1e-5) diff --git a/tests/run_tests.sh b/tests/run_tests.sh index df2f5fc..3121670 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -9,3 +9,5 @@ KERAS_BACKEND="torch" pytest test_wanda.py pytest test_keras_compression_layers.py DATA_FORMAT=channels_last pytest test_keras_compression_layers.py KERAS_BACKEND="torch" pytest test_torch_compression_layers.py +pytest test_torch_onnx_converter.py +pytest test_keras_onnx_converter.py diff --git a/tests/test_keras_onnx_converter.py b/tests/test_keras_onnx_converter.py new file mode 100644 index 0000000..91dc46f --- /dev/null +++ b/tests/test_keras_onnx_converter.py @@ -0,0 +1,197 @@ +"""Tests for the Keras β†’ ONNX converter (convert_to_onnx). + +Each test builds a small functional Keras model, runs a forward pass to +initialise all sublayer state, calls apply_final_compression, exports to ONNX +via convert_to_onnx(), and verifies that onnxruntime produces the same output +as the Keras model. + +bias=True/False is tested via parametrize where applicable. +""" + +import keras +import numpy as np +import pytest + +import pquant +from pquant.core.keras.convert_to_onnx import convert_to_onnx +from pquant.core.keras.layers import ( + PQBatchNormalization, + PQConv1d, + PQConv2d, + PQDense, + PQDepthwiseConv2d, + apply_final_compression, +) + +ort = pytest.importorskip("onnxruntime", reason="onnxruntime not installed") + +ATOL = 1e-4 + + +# --------------------------------------------------------------------------- +# fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def cfg(): + c = pquant.cs_config() + c.quantization_parameters.enable_quantization = False + return c + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + + +def _channels_first(): + return keras.backend.image_data_format() == "channels_first" + + +def _keras_out(model, x: np.ndarray) -> np.ndarray: + from keras import ops + + return ops.convert_to_numpy(model(x, training=False)) + + +def _onnx_run(model, x: np.ndarray, input_shape: tuple, tmp_path) -> np.ndarray: + path = str(tmp_path / "model.onnx") + convert_to_onnx(model, input_shape=input_shape, output_path=path) + sess = ort.InferenceSession(path) + in_name = sess.get_inputs()[0].name + return sess.run(None, {in_name: x})[0] + + +# --------------------------------------------------------------------------- +# PQDense +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("bias", [True, False]) +def test_dense_onnx(cfg, bias, tmp_path): + IN, OUT = 16, 8 + inputs = keras.Input(shape=(IN,)) + x = PQDense(cfg, units=OUT, use_bias=bias)(inputs) + model = keras.Model(inputs, x) + + dummy = np.zeros((1, IN), dtype=np.float32) + model(dummy) + apply_final_compression(model) + + x_np = np.random.randn(4, IN).astype(np.float32) + keras_out = _keras_out(model, x_np) + onnx_out = _onnx_run(model, x_np, input_shape=(IN,), tmp_path=tmp_path) + np.testing.assert_allclose(keras_out, onnx_out, atol=ATOL, err_msg=f"PQDense bias={bias}: keras vs ONNX mismatch") + + +# --------------------------------------------------------------------------- +# PQConv2d +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("bias", [True, False]) +def test_conv2d_onnx(cfg, bias, tmp_path): + IN_C, OUT_C, H, W = 3, 8, 8, 8 + if _channels_first(): + input_shape = (IN_C, H, W) + x_np = np.random.randn(2, IN_C, H, W).astype(np.float32) + else: + input_shape = (H, W, IN_C) + x_np = np.random.randn(2, H, W, IN_C).astype(np.float32) + + inputs = keras.Input(shape=input_shape) + x = PQConv2d(cfg, OUT_C, kernel_size=3, padding="same", use_bias=bias)(inputs) + model = keras.Model(inputs, x) + + dummy = np.zeros((1, *input_shape), dtype=np.float32) + model(dummy) + apply_final_compression(model) + + keras_out = _keras_out(model, x_np) + onnx_out = _onnx_run(model, x_np, input_shape=input_shape, tmp_path=tmp_path) + np.testing.assert_allclose(keras_out, onnx_out, atol=ATOL, err_msg=f"PQConv2d bias={bias}: keras vs ONNX mismatch") + + +# --------------------------------------------------------------------------- +# PQConv1d +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("bias", [True, False]) +def test_conv1d_onnx(cfg, bias, tmp_path): + IN_C, OUT_C, L = 4, 8, 16 + if _channels_first(): + input_shape = (IN_C, L) + x_np = np.random.randn(2, IN_C, L).astype(np.float32) + else: + input_shape = (L, IN_C) + x_np = np.random.randn(2, L, IN_C).astype(np.float32) + + inputs = keras.Input(shape=input_shape) + x = PQConv1d(cfg, OUT_C, kernel_size=3, padding="same", use_bias=bias)(inputs) + model = keras.Model(inputs, x) + + dummy = np.zeros((1, *input_shape), dtype=np.float32) + model(dummy) + apply_final_compression(model) + + keras_out = _keras_out(model, x_np) + onnx_out = _onnx_run(model, x_np, input_shape=input_shape, tmp_path=tmp_path) + np.testing.assert_allclose(keras_out, onnx_out, atol=ATOL, err_msg=f"PQConv1d bias={bias}: keras vs ONNX mismatch") + + +# --------------------------------------------------------------------------- +# PQBatchNormalization +# --------------------------------------------------------------------------- + + +def test_batchnorm_onnx(cfg, tmp_path): + IN_C, H, W = 8, 4, 4 + if _channels_first(): + input_shape = (IN_C, H, W) + x_np = np.random.randn(4, IN_C, H, W).astype(np.float32) + bn_axis = 1 + else: + input_shape = (H, W, IN_C) + x_np = np.random.randn(4, H, W, IN_C).astype(np.float32) + bn_axis = -1 + + inputs = keras.Input(shape=input_shape) + x = PQBatchNormalization(cfg, axis=bn_axis)(inputs) + model = keras.Model(inputs, x) + + dummy = np.zeros((1, *input_shape), dtype=np.float32) + model(dummy, training=True) # warm up running stats + apply_final_compression(model) + + keras_out = _keras_out(model, x_np) + onnx_out = _onnx_run(model, x_np, input_shape=input_shape, tmp_path=tmp_path) + np.testing.assert_allclose(keras_out, onnx_out, atol=ATOL, err_msg="PQBatchNormalization: keras vs ONNX mismatch") + + +# --------------------------------------------------------------------------- +# PQDepthwiseConv2d +# --------------------------------------------------------------------------- + + +def test_depthwise_conv2d_onnx(cfg, tmp_path): + IN_C, H, W = 4, 8, 8 + if _channels_first(): + input_shape = (IN_C, H, W) + x_np = np.random.randn(2, IN_C, H, W).astype(np.float32) + else: + input_shape = (H, W, IN_C) + x_np = np.random.randn(2, H, W, IN_C).astype(np.float32) + + inputs = keras.Input(shape=input_shape) + x = PQDepthwiseConv2d(cfg, kernel_size=3, padding="same")(inputs) + model = keras.Model(inputs, x) + + dummy = np.zeros((1, *input_shape), dtype=np.float32) + model(dummy) + apply_final_compression(model) + + keras_out = _keras_out(model, x_np) + onnx_out = _onnx_run(model, x_np, input_shape=input_shape, tmp_path=tmp_path) + np.testing.assert_allclose(keras_out, onnx_out, atol=ATOL, err_msg="PQDepthwiseConv2d: keras vs ONNX mismatch") diff --git a/tests/test_torch_compression_layers.py b/tests/test_torch_compression_layers.py index 3f2f80c..668c968 100644 --- a/tests/test_torch_compression_layers.py +++ b/tests/test_torch_compression_layers.py @@ -1,8 +1,8 @@ -import keras +import os + import numpy as np import pytest import torch -from keras import ops from torch import nn from torch.nn import ( AvgPool2d, @@ -14,10 +14,15 @@ Tanh, ) -from pquant import post_training_prune -from pquant.activations import PQActivation -from pquant.core.hyperparameter_optimization import PQConfig -from pquant.layers import ( +os.environ["KERAS_BACKEND"] = "torch" + +import keras # noqa: E402 +from keras import ops # noqa: E402 + +from pquant import post_training_prune # noqa: E402 +from pquant.activations import PQActivation # noqa: E402 +from pquant.core.hyperparameter_optimization import PQConfig # noqa: E402 +from pquant.layers import ( # noqa: E402 PQAvgPool1d, PQAvgPool2d, PQBatchNorm2d, diff --git a/tests/test_torch_onnx_converter.py b/tests/test_torch_onnx_converter.py new file mode 100644 index 0000000..2064d30 --- /dev/null +++ b/tests/test_torch_onnx_converter.py @@ -0,0 +1,384 @@ +"""Tests for convert_to_onnx / convert_to_onnx_fx. + +Each test builds a small model (one PQ layer + ReLU where applicable), runs a +forward pass to initialise any running statistics, calls apply_final_compression +on every PQ module, exports to ONNX with convert_to_onnx(), and then verifies +that onnxruntime produces the same output as the PyTorch model. + +The same check is repeated with bias=True and bias=False via parametrize. +""" + +import os + +import numpy as np +import pytest +import torch +import torch.nn as nn + +os.environ["KERAS_BACKEND"] = "torch" + +import pquant # noqa: E402 +from pquant.core.torch.convert_to_onnx import ( # noqa: E402 + convert_to_onnx, + convert_to_onnx_fx, + export_qdq_layernorm, +) +from pquant.layers import ( # noqa: E402 + PQAvgPool1d, + PQAvgPool2d, + PQBatchNorm1d, + PQBatchNorm2d, + PQConv1d, + PQConv2d, + PQDense, + PQMultiheadAttention, +) + +ort = pytest.importorskip("onnxruntime", reason="onnxruntime not installed") + +ATOL = 1e-4 # float32 Gemm/Conv can differ by ~1 ULP; keep some slack + + +# --------------------------------------------------------------------------- +# fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def cfg(): + c = pquant.cs_config() + c.quantization_parameters.enable_quantization = False + return c + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + + +def _apply_compression(model: nn.Module): + for m in model.modules(): + if hasattr(m, "apply_final_compression"): + m.apply_final_compression() + + +def _onnx_run(model: nn.Module, x: torch.Tensor, input_shape: tuple, tmp_path) -> np.ndarray: + """Export model β†’ ONNX file in tmp_path, run with onnxruntime, return output.""" + path = str(tmp_path / "model.onnx") + convert_to_onnx(model, input_shape=input_shape, output_path=path) + sess = ort.InferenceSession(path) + in_name = sess.get_inputs()[0].name + return sess.run(None, {in_name: x.cpu().numpy()})[0] + + +def _onnx_run_fx(model: nn.Module, x: torch.Tensor, input_shape: tuple, tmp_path) -> np.ndarray: + """FX-based export β†’ ONNX, run with onnxruntime.""" + path = str(tmp_path / "model_fx.onnx") + convert_to_onnx_fx(model, input_shape=input_shape, output_path=path) + sess = ort.InferenceSession(path) + in_name = sess.get_inputs()[0].name + return sess.run(None, {in_name: x.cpu().numpy()})[0] + + +def _torch_out(model: nn.Module, x: torch.Tensor) -> np.ndarray: + model.eval() + with torch.no_grad(): + return model(x).cpu().numpy() + + +# --------------------------------------------------------------------------- +# PQDense +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("bias", [True, False]) +def test_dense_onnx(cfg, bias, tmp_path): + IN, OUT = 16, 8 + model = nn.Sequential( + PQDense(cfg, in_features=IN, out_features=OUT, bias=bias), + nn.ReLU(), + ) + x = torch.randn(4, IN) + with torch.no_grad(): + model(x) # warm-up (needed for any running stats) + _apply_compression(model) + + torch_out = _torch_out(model, x) + onnx_out = _onnx_run(model, x, input_shape=(IN,), tmp_path=tmp_path) + np.testing.assert_allclose(torch_out, onnx_out, atol=ATOL, err_msg=f"PQDense bias={bias}: torch vs ONNX mismatch") + + +# --------------------------------------------------------------------------- +# PQConv2d +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("bias", [True, False]) +def test_conv2d_onnx(cfg, bias, tmp_path): + IN_C, OUT_C, H, W = 3, 8, 8, 8 + model = nn.Sequential( + PQConv2d(cfg, in_channels=IN_C, out_channels=OUT_C, kernel_size=3, padding=1, bias=bias), + nn.ReLU(), + ) + x = torch.randn(2, IN_C, H, W) + with torch.no_grad(): + model(x) + _apply_compression(model) + + torch_out = _torch_out(model, x) + onnx_out = _onnx_run(model, x, input_shape=(IN_C, H, W), tmp_path=tmp_path) + np.testing.assert_allclose(torch_out, onnx_out, atol=ATOL, err_msg=f"PQConv2d bias={bias}: torch vs ONNX mismatch") + + +# --------------------------------------------------------------------------- +# PQConv1d +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("bias", [True, False]) +def test_conv1d_onnx(cfg, bias, tmp_path): + IN_C, OUT_C, L = 4, 8, 16 + model = nn.Sequential( + PQConv1d(cfg, in_channels=IN_C, out_channels=OUT_C, kernel_size=3, padding=1, bias=bias), + nn.ReLU(), + ) + x = torch.randn(2, IN_C, L) + with torch.no_grad(): + model(x) + _apply_compression(model) + + torch_out = _torch_out(model, x) + onnx_out = _onnx_run(model, x, input_shape=(IN_C, L), tmp_path=tmp_path) + np.testing.assert_allclose(torch_out, onnx_out, atol=ATOL, err_msg=f"PQConv1d bias={bias}: torch vs ONNX mismatch") + + +# --------------------------------------------------------------------------- +# PQBatchNorm2d +# --------------------------------------------------------------------------- + + +def test_batchnorm2d_onnx(cfg, tmp_path): + C, H, W = 8, 4, 4 + model = nn.Sequential( + PQBatchNorm2d(cfg, num_features=C), + nn.ReLU(), + ) + x = torch.randn(4, C, H, W) + with torch.no_grad(): + model(x) + _apply_compression(model) + model.eval() # switch BN to use running stats + + torch_out = _torch_out(model, x) + onnx_out = _onnx_run(model, x, input_shape=(C, H, W), tmp_path=tmp_path) + np.testing.assert_allclose(torch_out, onnx_out, atol=ATOL, err_msg="PQBatchNorm2d: torch vs ONNX mismatch") + + +# --------------------------------------------------------------------------- +# PQBatchNorm1d +# --------------------------------------------------------------------------- + + +def test_batchnorm1d_onnx(cfg, tmp_path): + C, L = 8, 16 + model = nn.Sequential( + PQBatchNorm1d(cfg, num_features=C), + nn.ReLU(), + ) + x = torch.randn(4, C, L) + with torch.no_grad(): + model(x) + _apply_compression(model) + model.eval() + + torch_out = _torch_out(model, x) + onnx_out = _onnx_run(model, x, input_shape=(C, L), tmp_path=tmp_path) + np.testing.assert_allclose(torch_out, onnx_out, atol=ATOL, err_msg="PQBatchNorm1d: torch vs ONNX mismatch") + + +# --------------------------------------------------------------------------- +# PQAvgPool2d +# --------------------------------------------------------------------------- + + +def test_avgpool2d_onnx(cfg, tmp_path): + C, H, W = 8, 8, 8 + model = nn.Sequential( + PQAvgPool2d(cfg, kernel_size=2, stride=2), + ) + x = torch.randn(2, C, H, W) + with torch.no_grad(): + model(x) + _apply_compression(model) + + torch_out = _torch_out(model, x) + onnx_out = _onnx_run(model, x, input_shape=(C, H, W), tmp_path=tmp_path) + np.testing.assert_allclose(torch_out, onnx_out, atol=ATOL, err_msg="PQAvgPool2d: torch vs ONNX mismatch") + + +# --------------------------------------------------------------------------- +# PQAvgPool1d +# --------------------------------------------------------------------------- + + +def test_avgpool1d_onnx(cfg, tmp_path): + C, L = 8, 16 + model = nn.Sequential( + PQAvgPool1d(cfg, kernel_size=2, stride=2), + ) + x = torch.randn(2, C, L) + with torch.no_grad(): + model(x) + _apply_compression(model) + + torch_out = _torch_out(model, x) + onnx_out = _onnx_run(model, x, input_shape=(C, L), tmp_path=tmp_path) + np.testing.assert_allclose(torch_out, onnx_out, atol=ATOL, err_msg="PQAvgPool1d: torch vs ONNX mismatch") + + +# --------------------------------------------------------------------------- +# PQMultiheadAttention (uses FX converter; self-attention, batch_first=True) +# --------------------------------------------------------------------------- + + +class _SelfAttnModel(nn.Module): + """Thin wrapper so FX tracing sees a single-input model.""" + + def __init__(self, mha: PQMultiheadAttention): + super().__init__() + self.mha = mha + + def forward(self, x): + out, _ = self.mha(x, x, x) + return out + + +@pytest.mark.parametrize("bias", [True, False]) +def test_mha_onnx(cfg, bias, tmp_path): + E, H, T = 16, 4, 8 + mha = PQMultiheadAttention(cfg, embed_dim=E, num_heads=H, bias=bias, batch_first=True) + model = _SelfAttnModel(mha) + + x = torch.randn(2, T, E) + with torch.no_grad(): + model(x) + _apply_compression(model) + + torch_out = _torch_out(model, x) + onnx_out = _onnx_run_fx(model, x, input_shape=(T, E), tmp_path=tmp_path) + np.testing.assert_allclose( + torch_out, onnx_out, atol=ATOL, err_msg=f"PQMultiheadAttention bias={bias}: torch vs ONNX mismatch" + ) + + +# --------------------------------------------------------------------------- +# Static-QDQ LayerNormalization graph +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("input_shape", [(4, 64), (1, 4, 64)]) +def test_qdq_layernorm_export(input_shape, tmp_path): + import onnx + + D = input_shape[-1] + rng = np.random.default_rng(0) + # Q7 representable: gamma = k / 128, k integer, |k| < 32768 + gamma_q = rng.integers(low=64, high=192, size=(D,), dtype=np.int32) # ~0.5 .. 1.5 + gamma = (gamma_q.astype(np.float32)) / (1 << 7) + # Q15 representable: beta = k / 32768, k integer, |k| < 32768 (so |beta| < 1) + beta_q = rng.integers(low=-1024, high=1024, size=(D,), dtype=np.int32) + beta = (beta_q.astype(np.float32)) / (1 << 15) + + input_scale_log2 = -7 # input_scale = 2**-7 + output_scale_log2 = -6 # output_scale = 2**-6 + eps_q0 = 1 + + path = str(tmp_path / "qdq_layernorm.onnx") + model_proto = export_qdq_layernorm( + output_path=path, + input_shape=input_shape, + gamma=gamma, + beta=beta, + input_scale_log2=input_scale_log2, + output_scale_log2=output_scale_log2, + eps_q0=eps_q0, + ) + + # ----- structural checks ----- + op_types = [n.op_type for n in model_proto.graph.node] + assert op_types == ["DequantizeLinear", "LayerNormalization", "QuantizeLinear", "DequantizeLinear"] + + ln_node = model_proto.graph.node[1] + axis = next(a.i for a in ln_node.attribute if a.name == "axis") + eps_attr = next(a.f for a in ln_node.attribute if a.name == "epsilon") + assert axis == -1 + expected_eps = eps_q0 * (2.0**input_scale_log2) ** 2 + assert abs(eps_attr - expected_eps) < 1e-12 + + # input must be int8, output float + assert len(model_proto.graph.input) == 1 + assert model_proto.graph.input[0].type.tensor_type.elem_type == onnx.TensorProto.INT8 + assert model_proto.graph.output[0].type.tensor_type.elem_type == onnx.TensorProto.FLOAT + in_dims = [d.dim_value for d in model_proto.graph.input[0].type.tensor_type.shape.dim] + assert tuple(in_dims) == input_shape + + # zero-points must be int8 zero + inits = {t.name: t for t in model_proto.graph.initializer} + for zp_name in ("input_zero_point", "output_zero_point"): + zp = onnx.numpy_helper.to_array(inits[zp_name]) + assert zp.dtype == np.int8 + assert int(zp) == 0 + + # scales must be exact powers of two + in_scale = float(onnx.numpy_helper.to_array(inits["input_scale"])) + out_scale = float(onnx.numpy_helper.to_array(inits["output_scale"])) + assert in_scale == 2.0**input_scale_log2 + assert out_scale == 2.0**output_scale_log2 + + # ----- numerical check via onnxruntime ----- + sess = ort.InferenceSession(path) + in_name = sess.get_inputs()[0].name + x_q = rng.integers(low=-64, high=64, size=input_shape, dtype=np.int8) + onnx_out = sess.run(None, {in_name: x_q})[0] + + # Reference: dequantize -> layernorm(axis=-1) -> quantize -> dequantize + x_f = x_q.astype(np.float32) * in_scale + mean = x_f.mean(axis=-1, keepdims=True) + var = x_f.var(axis=-1, keepdims=True) + x_norm = (x_f - mean) / np.sqrt(var + expected_eps) + y_f = x_norm * gamma + beta + y_q = np.clip(np.round(y_f / out_scale), -128, 127).astype(np.int8) + y_ref = y_q.astype(np.float32) * out_scale + + np.testing.assert_allclose(onnx_out, y_ref, atol=out_scale * 0.5) + + +def test_qdq_layernorm_validation(tmp_path): + path = str(tmp_path / "bad.onnx") + D = 64 + gamma = np.ones(D, dtype=np.float32) + beta = np.zeros(D, dtype=np.float32) + + # rank-1 input: rejected + with pytest.raises(ValueError, match="rank"): + export_qdq_layernorm(path, (D,), gamma, beta, -7, -6) + + # last dim not multiple of 32 + with pytest.raises(ValueError, match="multiple of 32"): + export_qdq_layernorm(path, (4, 16), np.ones(16, np.float32), np.zeros(16, np.float32), -7, -6) + + # last dim not power of two (96 = 32*3) + with pytest.raises(ValueError, match="power of two"): + export_qdq_layernorm(path, (4, 96), np.ones(96, np.float32), np.zeros(96, np.float32), -7, -6) + + # gamma not Q7-representable (1/3 is not k/128 exactly) + with pytest.raises(ValueError, match="gamma"): + export_qdq_layernorm(path, (4, D), np.full(D, 1.0 / 3.0, np.float32), beta, -7, -6) + + # beta not Q15-representable (1/3 is not k/32768 exactly) + with pytest.raises(ValueError, match="beta"): + export_qdq_layernorm(path, (4, D), gamma, np.full(D, 1.0 / 3.0, np.float32), -7, -6) + + # eps_q0 < 1 + with pytest.raises(ValueError, match="eps_q0"): + export_qdq_layernorm(path, (4, D), gamma, beta, -7, -6, eps_q0=0) From 158f4e89494427f5bb4eff9523da860d53a6fe9a Mon Sep 17 00:00:00 2001 From: nroope Date: Fri, 12 Jun 2026 12:53:51 +0200 Subject: [PATCH 08/22] Add alkaid interface (#45) * initial alkaid interface * Refactored PQ MHA quantizer flow, added HGQ style quantized Softmax * Added an upper limit to integer bits in dynamic data quantization, could previously go over total bitwidth --- pyproject.toml | 3 + src/pquant/_alkaid_plugin/__init__.py | 0 src/pquant/_alkaid_plugin/_alkaid_common.py | 92 +++++ .../_alkaid_plugin/_alkaid_keras_plugin.py | 295 +++++++++++++ .../_alkaid_plugin/_alkaid_torch_plugin.py | 297 +++++++++++++ src/pquant/core/keras/activations.py | 264 +++++++++++- src/pquant/core/keras/convert_to_onnx.py | 43 +- src/pquant/core/keras/layers.py | 148 +++---- src/pquant/core/keras/quantizer.py | 1 + src/pquant/core/torch/activations.py | 218 +++++++++- src/pquant/core/torch/convert_to_onnx.py | 41 +- src/pquant/core/torch/layers.py | 157 +++---- src/pquant/core/torch/quantizer.py | 1 + tests/test_keras_alkaid_conversion.py | 379 +++++++++++++++++ tests/test_torch_alkaid_conversion.py | 390 ++++++++++++++++++ 15 files changed, 2089 insertions(+), 240 deletions(-) create mode 100644 src/pquant/_alkaid_plugin/__init__.py create mode 100644 src/pquant/_alkaid_plugin/_alkaid_common.py create mode 100644 src/pquant/_alkaid_plugin/_alkaid_keras_plugin.py create mode 100644 src/pquant/_alkaid_plugin/_alkaid_torch_plugin.py create mode 100644 tests/test_keras_alkaid_conversion.py create mode 100644 tests/test_torch_alkaid_conversion.py diff --git a/pyproject.toml b/pyproject.toml index fb0c210..aaa2530 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,9 @@ optional-dependencies.test = [ "pytest>=8.4" ] optional-dependencies.torch = [ "torch>=2.1" ] urls.repository = "https://github.com/cern-nextgen/PQuantML" +entry-points."alkaid_keras".pquant = "pquant._alkaid_plugin._alkaid_keras_plugin:register" +entry-points."alkaid_torch".pquant = "pquant._alkaid_plugin._alkaid_torch_plugin:register" + [tool.setuptools] packages = [ "pquant" ] include-package-data = true diff --git a/src/pquant/_alkaid_plugin/__init__.py b/src/pquant/_alkaid_plugin/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pquant/_alkaid_plugin/_alkaid_common.py b/src/pquant/_alkaid_plugin/_alkaid_common.py new file mode 100644 index 0000000..687dba9 --- /dev/null +++ b/src/pquant/_alkaid_plugin/_alkaid_common.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from typing import Any + +import numpy as np +from alkaid.trace.ops import quantize as alkaid_quantize + + +class PQuantAlkaidError(ValueError): + """Raised for PQuant states that cannot be replayed by Alkaid.""" + + +def to_numpy(value: Any) -> np.ndarray: + if value is None: + return np.array(0.0) + if isinstance(value, np.ndarray): + return value + if hasattr(value, 'detach'): + value = value.detach() + if hasattr(value, 'cpu'): + value = value.cpu() + return value.numpy() + try: + import keras + + return np.asarray(keras.ops.convert_to_numpy(value)) + except Exception: + return np.asarray(value) + + +def to_bool(value: Any, default: bool = False) -> bool: + if value is None: + return default + try: + arr = to_numpy(value) + except Exception: + return bool(value) + if arr.shape == (): + return bool(arr.item()) + return bool(np.all(arr)) + + +def to_int_bits(value: Any) -> np.ndarray: + return np.rint(to_numpy(value)).astype(np.int64) + + +def raw_module_attr(obj: Any, name: str, default: Any = None) -> Any: + for storage_name in ('_parameters', '_buffers', '_modules'): + storage = getattr(obj, storage_name, None) + if isinstance(storage, dict) and name in storage: + return storage[name] + try: + return object.__getattribute__(obj, name) + except AttributeError: + return getattr(obj, name, default) + + +def quantizer_kif(quantizer: Any) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + if hasattr(quantizer, '_parameters'): + if not bool(raw_module_attr(quantizer, 'use_hgq', False)): + return ( + to_int_bits(raw_module_attr(quantizer, 'k')), + to_int_bits(raw_module_attr(quantizer, 'i')), + to_int_bits(raw_module_attr(quantizer, 'f')), + ) + inner = raw_module_attr(quantizer, 'quantizer') + if hasattr(inner, '_parameters') or hasattr(inner, '_buffers'): + k = raw_module_attr(inner, '_k') + i = raw_module_attr(inner, '_i_raw', None) + if i is None: + i = raw_module_attr(inner, '_i') + f = raw_module_attr(inner, '_f') + return to_int_bits(k), to_int_bits(i), to_int_bits(f) + k, i, f = quantizer.get_quantization_bits() + return to_int_bits(k), to_int_bits(i), to_int_bits(f) + + +def replay_quantizer(quantizer: Any, x: Any) -> Any: + k, i, f = quantizer_kif(quantizer) + inner = raw_module_attr(quantizer, 'quantizer', None) + overflow = raw_module_attr(quantizer, 'overflow', raw_module_attr(inner, 'overflow_mode', 'WRAP')) + round_mode = raw_module_attr(quantizer, 'round_mode', raw_module_attr(inner, 'round_mode', 'TRN')) + return alkaid_quantize(x, k=k, i=i, f=f, overflow_mode=str(overflow).upper(), round_mode=str(round_mode).upper()) + + +def replay_quantizer_if_enabled(layer: Any, quantizer_name: str, x: Any, flag_name: str) -> Any: + if not bool(getattr(layer, 'enable_quantization', True)): + return x + if not bool(getattr(layer, flag_name, True)): + return x + quantizer = getattr(layer, quantizer_name, None) + return replay_quantizer(quantizer, x) diff --git a/src/pquant/_alkaid_plugin/_alkaid_keras_plugin.py b/src/pquant/_alkaid_plugin/_alkaid_keras_plugin.py new file mode 100644 index 0000000..4e75957 --- /dev/null +++ b/src/pquant/_alkaid_plugin/_alkaid_keras_plugin.py @@ -0,0 +1,295 @@ +from __future__ import annotations + +from math import prod + +import keras +import numpy as np +from alkaid.converter.builtin.keras.layers._base import ReplayOperationBase +from alkaid.converter.builtin.keras.layers.activation import keras_numpy_unary_map +from alkaid.converter.builtin.keras.layers.batchnorm import ReplayBatchNormalization +from alkaid.converter.builtin.keras.layers.conv import _conv +from alkaid.converter.builtin.keras.layers.pool import ReplayPool +from alkaid.trace import FVArray +from alkaid.trace.ops import einsum, extract_patches +from keras.layers import DepthwiseConv1D, DepthwiseConv2D + +from pquant._alkaid_plugin._alkaid_common import ( + PQuantAlkaidError, + replay_quantizer, + replay_quantizer_if_enabled, + to_bool, + to_numpy, +) +from pquant.core.keras.activations import PQActivation +from pquant.core.keras.layers import ( + PQAvgPool1d, + PQAvgPool2d, + PQBatchNormalization, + PQConv1d, + PQConv2d, + PQDense, + PQDepthwiseConv2d, + PQMultiheadAttention, + PQSeparableConv2d, + PQSoftmax, +) +from pquant.core.keras.quantizer import Quantizer + + +def _assert_final_compression(layer) -> None: + if not to_bool(getattr(layer, 'final_compression_done', False)): + raise PQuantAlkaidError( + f'{layer.__class__.__name__} must have apply_final_compression() applied before Alkaid conversion.' + ) + + +def _weight(layer) -> np.ndarray: + _assert_final_compression(layer) + return to_numpy(layer._kernel) + + +def _bias(layer) -> np.ndarray: + _assert_final_compression(layer) + bias = getattr(layer, '_bias', None) + if bias is None: + return np.array(0.0) + return to_numpy(bias) + + +class ReplayPQuantQuantizer(ReplayOperationBase): + __activation_handled__ = True + handles = (Quantizer,) + + def call(self, x: FVArray) -> FVArray: + return replay_quantizer(self.op, x) + + +class ReplayPQuantDense(ReplayOperationBase): + handles = (PQDense,) + + def call(self, inputs: FVArray) -> FVArray: + layer = self.op + inputs = replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + out = np.einsum('...c,cC->...C', inputs, _weight(layer)) + _bias(layer) + return replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + + +class ReplayPQuantConv(ReplayOperationBase): + handles = (PQConv1d, PQConv2d, PQDepthwiseConv2d) + + def call(self, inputs: FVArray) -> FVArray: + layer = self.op + inputs = replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + kernel = _weight(layer) + bias = _bias(layer) + + if isinstance(layer, (DepthwiseConv1D, DepthwiseConv2D)): + ch_in, dm = kernel.shape[-2:] + kernel = kernel.reshape(*kernel.shape[:-2], 1, ch_in * dm) + groups = ch_in + else: + groups = layer.groups + + x = extract_patches( + inputs, + size=layer.kernel_size, + strides=layer.strides, + dilation_rate=layer.dilation_rate, + padding=layer.padding, + data_format=layer.data_format, + ) + ch_out = kernel.shape[-1] + ch_in_per_g = kernel.shape[-2] + k_vol = int(prod(layer.kernel_size)) + out = _conv( + x, + kernel, + k_vol=k_vol, + groups=groups, + ch_in_per_g=ch_in_per_g, + out_per_g=ch_out // groups, + ) + if bias.shape != (): + out = out + bias + if layer.data_format == 'channels_first': + out = np.moveaxis(out, -1, 1) # type: ignore + return replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + + +class ReplayPQuantSeparableConv(ReplayOperationBase): + handles = (PQSeparableConv2d,) + + def call(self, inputs: FVArray) -> FVArray: + layer = self.op + x = ReplayPQuantConv(layer.depthwise_conv).call(inputs) + return ReplayPQuantConv(layer.pointwise_conv).call(x) + + +class ReplayPQuantBatchNormalization(ReplayBatchNormalization): + handles = (PQBatchNormalization,) + + def fused_scale_offset(self) -> tuple[np.ndarray, np.ndarray]: + layer = self.op + _assert_final_compression(layer) + mean = to_numpy(keras.ops.cast(layer.moving_mean, layer.dtype)) + variance = to_numpy(keras.ops.cast(layer.moving_variance, layer.dtype)) + if layer.scale: + gamma = to_numpy(keras.ops.cast(layer.gamma, layer.dtype)) + else: + gamma = np.ones_like(mean) + if layer.center: + beta = to_numpy(keras.ops.cast(layer.beta, layer.dtype)) + else: + beta = np.zeros_like(mean) + scale = gamma / np.sqrt(variance + layer.epsilon) + offset = beta - mean * scale + return scale, offset + + def call(self, inputs: FVArray, mask=None) -> FVArray: + layer = self.op + inputs = replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + scale, offset = self.fused_scale_offset() + shape = [1] * inputs.ndim + axis = layer.axis if isinstance(layer.axis, (list, tuple)) else [layer.axis] + for a in axis: + aa = a if a >= 0 else inputs.ndim + a + shape[aa] = inputs.shape[aa] + out = inputs + if not np.all(scale == 1): + out = out * scale.reshape(shape) # type: ignore + if not np.all(offset == 0): + out = out + offset.reshape(shape) # type: ignore + return out + + +class ReplayPQuantAvgPool(ReplayPool): + __activation_handled__ = True + handles = (PQAvgPool1d, PQAvgPool2d) + + def call(self, inputs: FVArray, mask: None = None) -> FVArray: + layer = self.op + inputs = replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + out = super().call(inputs, mask=mask) + return replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + + +class ReplayPQuantActivation(ReplayOperationBase): + __activation_handled__ = True + handles = (PQActivation,) + + def call(self, inputs: FVArray) -> FVArray: + layer = self.op + if ( + not bool(getattr(layer, 'use_hgq', False)) + and bool(getattr(layer, 'use_multiplier', False)) + and layer.activation_name == 'relu' + and hasattr(layer, 'multiplier') + ): + inputs = inputs * (2.0 ** np.rint(to_numpy(layer.multiplier))) + inputs = replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + if layer.activation_name not in keras_numpy_unary_map: + raise PQuantAlkaidError(f'Unsupported PQuant activation for Alkaid conversion: {layer.activation_name!r}') + out = keras_numpy_unary_map[layer.activation_name](inputs) + return replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + + +def _table_fn(table): + """Numpy-callable for a PQActivation lookup table, evaluated in float32 like the keras runtime.""" + fn = table.activation_function + + def apply_fn(v: np.ndarray) -> np.ndarray: + t = keras.ops.cast(keras.ops.convert_to_tensor(v), 'float32') + return np.asarray(keras.ops.convert_to_numpy(fn(t)), dtype=np.float64) + + return apply_fn + + +class ReplayPQuantSoftmax(ReplayOperationBase): + __activation_handled__ = True + handles = (PQSoftmax,) + + @staticmethod + def _replay_table(table, x: FVArray) -> FVArray: + if not (table.quantize_output and table.enable_quantization): + raise PQuantAlkaidError( + f'PQSoftmax table {table.name!r} must have an enabled output quantizer for Alkaid conversion.' + ) + x = replay_quantizer_if_enabled(table, 'input_quantizer', x, 'quantize_input') + out = x.apply(_table_fn(table)) + return replay_quantizer(table.output_quantizer, out) + + def call(self, inputs: FVArray, mask=None) -> FVArray: + layer = self.op + if mask is not None: + raise PQuantAlkaidError('PQSoftmax masks are not supported in Alkaid conversion.') + inputs = replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + if layer.stable: + inputs = np.max(inputs, axis=layer.axes, keepdims=True) - inputs # type: ignore + exp_inp = self._replay_table(layer.exp_table, inputs) + sums = np.sum(exp_inp, axis=layer.axes, keepdims=True) + divisor = self._replay_table(layer.inv_table, sums) + out = exp_inp * divisor + return replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + + +class ReplayPQuantMultiheadAttention(ReplayOperationBase): + __activation_handled__ = True + handles = (PQMultiheadAttention,) + + def call(self, inputs, key_padding_mask=None, attn_mask=None, need_weights=True): + layer = self.op + if key_padding_mask is not None or attn_mask is not None: + raise PQuantAlkaidError('Attention masks are not supported in Alkaid conversion.') + + if isinstance(inputs, (list, tuple)): + if len(inputs) == 3: + query, key, value = inputs + elif len(inputs) == 2: + query, key = inputs + value = key + else: + query = key = value = inputs[0] + else: + query = key = value = inputs + + batch_size, query_len = query.shape[0], query.shape[1] + key_len = key.shape[1] + num_heads, head_dim = layer.num_heads, layer.head_dim + + q = ReplayPQuantDense(layer.q_proj).call(query) # (B, T, E) + k = ReplayPQuantDense(layer.k_proj).call(key) # (B, S, E) + v = ReplayPQuantDense(layer.v_proj).call(value) # (B, S, E) + + # Reshape to (B, H, T/S, head_dim) + q = q.reshape(batch_size, query_len, num_heads, head_dim).transpose(0, 2, 1, 3) + k = k.reshape(batch_size, key_len, num_heads, head_dim).transpose(0, 2, 1, 3) + v = v.reshape(batch_size, key_len, num_heads, head_dim).transpose(0, 2, 1, 3) + + scale = float(np.float32(layer.scale)) + attn_scores = einsum('bhtd,bhsd->bhts', q, k) * scale + + # The softmax's own input/output quantizers handle the scores and the attention weights + attn_weights = ReplayPQuantSoftmax(layer.softmax).call(attn_scores) + + # Weighted sum of values (dropout is an inference no-op): (B, H, T, head_dim) + out = einsum('bhts,bhsd->bhtd', attn_weights, v) + + # Merge heads: (B, T, E) + out = out.transpose(0, 2, 1, 3).reshape(batch_size, query_len, layer.embed_dim) + out = ReplayPQuantDense(layer.out_proj).call(out) + + if need_weights: + # Average attention weights over heads: (B, T, S) + return out, np.mean(attn_weights, axis=1) + return (out,) + + +def register() -> None: + """Entry point for Alkaid's ``alkaid_keras`` second-level plugin group.""" + try: + from alkaid.converter import _plugin_loader + + _plugin_loader._LOADED.add(('pquant', 'keras')) + except Exception: + pass + return None diff --git a/src/pquant/_alkaid_plugin/_alkaid_torch_plugin.py b/src/pquant/_alkaid_plugin/_alkaid_torch_plugin.py new file mode 100644 index 0000000..22428e6 --- /dev/null +++ b/src/pquant/_alkaid_plugin/_alkaid_torch_plugin.py @@ -0,0 +1,297 @@ +from __future__ import annotations + +import builtins +import operator +from functools import wraps +from typing import Any + +import numpy as np +import torch + +try: + from torch.fx._symbolic_trace import is_fx_symbolic_tracing +except ImportError: # torch < 2.8 exposes it as is_fx_tracing + from torch.fx._symbolic_trace import is_fx_tracing as is_fx_symbolic_tracing +from alkaid.converter.builtin.torch.layers.functional import _functional_map +from alkaid.converter.builtin.torch.layers.methods import _method_map +from alkaid.converter.builtin.torch.layers.modules import ReplayModuleBase +from alkaid.trace import FVArray + +from pquant._alkaid_plugin._alkaid_common import ( + PQuantAlkaidError, + replay_quantizer, + replay_quantizer_if_enabled, +) +from pquant.core.torch.activations import PQActivation +from pquant.core.torch.layers import ( + PQBatchNorm1d, + PQBatchNorm2d, + PQConv1d, + PQConv2d, + PQDense, + PQSoftmax, + PQWeightBiasBase, +) +from pquant.core.torch.quantizer import Quantizer + + +def _contains_fx_proxy(value: Any) -> bool: + from torch.fx.proxy import Proxy + + if isinstance(value, Proxy): + return True + if isinstance(value, (tuple, list)): + return any(_contains_fx_proxy(v) for v in value) + if isinstance(value, dict): + return any(_contains_fx_proxy(v) for v in value.values()) + return False + + +def _patch_once(cls: type, name: str, wrapper_factory) -> None: + marker = f'__alkaid_pquant_patched_{name}__' + if getattr(cls, marker, False): + return + original = getattr(cls, name) + setattr(cls, f'__alkaid_pquant_original_{name}__', original) + setattr(cls, name, wrapper_factory(original)) + setattr(cls, marker, True) + + +def _module_parameter(module: torch.nn.Module, name: str) -> Any: + if name in module._parameters: + return module._parameters[name] + return getattr(module, name) + + +def _module_bool(module: torch.nn.Module, name: str, default: bool = False) -> bool: + value = module._parameters.get(name, getattr(module, name, default)) + if isinstance(value, torch.Tensor): + return bool(value.detach().cpu().item()) + return bool(value) + + +def _assert_final_compression(module: torch.nn.Module) -> None: + if not _module_bool(module, 'final_compression_done'): + raise PQuantAlkaidError( + f'{type(module).__name__} must have apply_final_compression() applied before Alkaid conversion.' + ) + + +def _patch_weight_bias_properties() -> None: + for cls in (PQDense, PQConv1d, PQConv2d, PQBatchNorm1d, PQBatchNorm2d): + marker = '__alkaid_pquant_patched_weight_bias__' + if getattr(cls, marker, False): + continue + original_weight = cls.weight.fget + original_bias = cls.bias.fget + + def weight(self, _original_weight=original_weight): + if not is_fx_symbolic_tracing(): + return _original_weight(self) + _assert_final_compression(self) + return _module_parameter(self, '_weight') + + def bias(self, _original_bias=original_bias): + if not is_fx_symbolic_tracing(): + return _original_bias(self) + _assert_final_compression(self) + return _module_parameter(self, '_bias') + + cls.weight = property(weight) + cls.bias = property(bias) + setattr(cls, marker, True) + + +def _patch_lazy_build_assertions() -> None: + def wrap_pre_forward(original): + @wraps(original) + def wrapped(self, x): + if not _contains_fx_proxy(x): + return original(self, x) + if self.quantize_input: + x = self.quantize(x, self.input_quantizer) + return x + + return wrapped + + _patch_once(PQWeightBiasBase, 'pre_forward', wrap_pre_forward) + + def wrap_pre_activation(original): + @wraps(original) + def wrapped(self, x): + if not _contains_fx_proxy(x): + return original(self, x) + if not self.use_hgq and self.use_multiplier and self.activation_name == 'relu' and hasattr(self, 'multiplier'): + multiplier = _module_parameter(self, 'multiplier') + x = x * (2.0 ** torch.round(multiplier.detach()).item()) + if self.quantize_input and self.enable_quantization: + x = self.input_quantizer(x) + return x + + return wrapped + + _patch_once(PQActivation, 'pre_activation', wrap_pre_activation) + + def wrap_bn_forward(original): + @wraps(original) + def wrapped(self, input): + if not _contains_fx_proxy(input): + return original(self, input) + if self.quantize_input and self.enable_quantization: + input = self.input_quantizer(input) + return torch.nn.functional.batch_norm( + input, + self.running_mean, + self.running_var, + self.weight, + self.bias, + False, + self.momentum, + self.eps, + ) + + return wrapped + + _patch_once(PQBatchNorm1d, 'forward', wrap_bn_forward) + _patch_once(PQBatchNorm2d, 'forward', wrap_bn_forward) + + +class ReplayPQuantQuantizer(ReplayModuleBase): + handles = (Quantizer,) + + def call(self, input: FVArray) -> FVArray: + return replay_quantizer(self.module, input) + + +def _table_fn(table): + """Numpy-callable for a PQActivation lookup table, evaluated in float32 like the torch runtime.""" + fn = table.activation_function + + def apply_fn(v: np.ndarray) -> np.ndarray: + with torch.no_grad(): + t = torch.as_tensor(v, dtype=torch.float32, device='cpu') + return fn(t).detach().cpu().numpy().astype(np.float64) + + return apply_fn + + +class ReplayPQuantSoftmax(ReplayModuleBase): + """Replay PQSoftmax as a single fx-leaf module""" + + handles = (PQSoftmax,) + + @staticmethod + def _replay_table(table, x: FVArray) -> FVArray: + if not (table.quantize_output and table.enable_quantization): + raise PQuantAlkaidError( + f'PQSoftmax table {type(table).__name__} must have an enabled output quantizer for Alkaid conversion.' + ) + x = replay_quantizer_if_enabled(table, 'input_quantizer', x, 'quantize_input') + out = x.apply(_table_fn(table)) + return replay_quantizer(table.output_quantizer, out) + + def call(self, inputs: FVArray, mask=None) -> FVArray: + module = self.module + if mask is not None: + raise PQuantAlkaidError('PQSoftmax masks are not supported in Alkaid conversion.') + if not module.built: + raise PQuantAlkaidError('PQSoftmax must be built (one real forward) before Alkaid conversion.') + inputs = replay_quantizer_if_enabled(module, 'input_quantizer', inputs, 'quantize_input') + if module.stable: + inputs = np.max(inputs, axis=module.axes, keepdims=True) - inputs # type: ignore + exp_inp = self._replay_table(module.exp_table, inputs) + sums = np.sum(exp_inp, axis=module.axes, keepdims=True) + divisor = self._replay_table(module.inv_table, sums) + out = exp_inp * divisor + return replay_quantizer_if_enabled(module, 'output_quantizer', out, 'quantize_output') + + +def _patch_root_quantizer_trace() -> None: + import alkaid.converter.builtin.torch.main as torch_main + + tracer_cls = torch_main.TorchALIRTracer + marker = '__alkaid_pquant_patched_root_quantizer__' + if getattr(tracer_cls, marker, False): + return + original = tracer_cls.apply_model + + @wraps(original) + def wrapped(self, verbose: bool, inputs: tuple[FVArray, ...]): + if isinstance(self.model, Quantizer): + if isinstance(inputs, FVArray): + inputs = (inputs,) + replay = ReplayPQuantQuantizer(self.model) + dump = replay(*inputs) + return {'inputs': tuple(inputs), 'quantizer/final': dump['final'], 'final': dump['final']}, ['final'] + return original(self, verbose, inputs) + + tracer_cls.apply_model = wrapped + setattr(tracer_cls, marker, True) + + +def _replay_getattr(obj: Any, name: str, *default: Any) -> Any: + if default: + return getattr(obj, name, default[0]) + return getattr(obj, name) + + +def _tensor(data: Any, *args: Any, **kwargs: Any) -> Any: + if isinstance(data, FVArray): + return data + if isinstance(data, torch.Tensor): + data = data.detach().cpu().numpy() + return np.asarray(data) + + +def _normalize_shape(args: tuple[Any, ...]) -> tuple[int, ...]: + if len(args) == 1 and isinstance(args[0], (tuple, list, torch.Size)): + return tuple(int(v) for v in args[0]) + return tuple(int(v) for v in args) + + +def _zeros(*size: Any, **kwargs: Any) -> np.ndarray: + return np.zeros(_normalize_shape(size), dtype=np.float32) + + +def _ones(*size: Any, **kwargs: Any) -> np.ndarray: + return np.ones(_normalize_shape(size), dtype=np.float32) + + +def _full(size: Any, fill_value: Any, **kwargs: Any) -> np.ndarray: + return np.full(_normalize_shape((size,)), fill_value, dtype=np.float32) + + +def _zeros_like(x: Any, **kwargs: Any) -> np.ndarray: + return np.zeros(tuple(x.shape), dtype=np.float32) + + +def _ones_like(x: Any, **kwargs: Any) -> np.ndarray: + return np.ones(tuple(x.shape), dtype=np.float32) + + +def _register_functional_helpers() -> None: + _functional_map.setdefault(operator.pow, lambda a, b: a**b) + _functional_map.setdefault(torch.pow, lambda a, b: a**b) + _functional_map.setdefault(builtins.getattr, _replay_getattr) + _functional_map.setdefault(torch.tensor, _tensor) + _functional_map.setdefault(torch.as_tensor, _tensor) + _functional_map.setdefault(torch.zeros, _zeros) + _functional_map.setdefault(torch.ones, _ones) + _functional_map.setdefault(torch.full, _full) + _functional_map.setdefault(torch.zeros_like, _zeros_like) + _functional_map.setdefault(torch.ones_like, _ones_like) + _method_map.setdefault('pow', lambda receiver, exponent, **_kwargs: receiver**exponent) + + +def register() -> None: + """Entry point for Alkaid's ``alkaid_torch`` second-level plugin group.""" + _patch_lazy_build_assertions() + _patch_weight_bias_properties() + _patch_root_quantizer_trace() + _register_functional_helpers() + try: + from alkaid.converter import _plugin_loader + + _plugin_loader._LOADED.add(('pquant', 'torch')) + except Exception: + pass diff --git a/src/pquant/core/keras/activations.py b/src/pquant/core/keras/activations.py index 5ae4542..f837366 100644 --- a/src/pquant/core/keras/activations.py +++ b/src/pquant/core/keras/activations.py @@ -1,7 +1,10 @@ +from math import prod from typing import Tuple from typing import TypeVar as T +from typing import Union import keras +from keras import ops from keras.ops import maximum, minimum, relu, tanh from pquant.core.keras.quantizer import Quantizer @@ -33,6 +36,7 @@ def __init__( out_quant_bits: Tuple[T, T, T] = None, quantize_input=True, quantize_output=False, + enable_ebops=True, **kwargs, ): super().__init__(**kwargs) @@ -55,8 +59,14 @@ def __init__( self.k_output, self.i_output, self.f_output = out_quant_bits self.in_quant_bits = in_quant_bits self.out_quant_bits = out_quant_bits - self.activation_name = activation.lower() - self.activation_function = activation_registry.get(self.activation_name) + if isinstance(activation, str): + self.activation_name = activation.lower() + self.activation_function = activation_registry.get(self.activation_name) + else: + # A callable was passed directly instead of a registry key (e.g. the exp/inv + # lookup-table functions used by QSoftmax). + self.activation_function = activation + self.activation_name = getattr(activation, "__name__", activation.__class__.__name__).lower() self.config = config self.enable_quantization = config.quantization_parameters.enable_quantization self.use_hgq = config.quantization_parameters.use_high_granularity_quantization @@ -74,6 +84,7 @@ def __init__( self.saved_inputs = [] self.quantize_input = quantize_input self.quantize_output = quantize_output + self.enable_ebops = enable_ebops self.built = False def build(self, input_shape): @@ -130,9 +141,11 @@ def post_pre_train_function(self): self.output_quantizer.post_pre_train_function() def ebops(self): + if not self.enable_ebops: + return 0.0 if self.quantize_input and self.quantize_output: - bw_inp = self.input_quantizer.quantizer.bits_(self.input_shape) - bw_out = self.output_quantizer.quantizer.bits_(self.input_shape) + bw_inp = self.input_quantizer.get_total_bits(self.input_shape) + bw_out = self.output_quantizer.get_total_bits(self.input_shape) return keras.ops.sum((2.0**bw_inp) * bw_out) * 1e-4 # type: ignore return 0.0 @@ -178,6 +191,7 @@ def get_config(self): "config": self.config.get_dict(), "quantize_input": self.quantize_input, "quantize_output": self.quantize_output, + "enable_ebops": self.enable_ebops, "activation": self.activation_name, "in_quant_bits": self.in_quant_bits, "out_quant_bits": self.out_quant_bits, @@ -187,3 +201,245 @@ def get_config(self): def extra_repr(self): return f"quantize_input = {self.quantize_input}, quantize_output = {self.quantize_output}" + + +@keras.saving.register_keras_serializable(package="PQuantML") +class PQSoftmax(keras.layers.Layer): + """Quantized softmax that mirrors HGQ's ``QSoftmax``. + + Args: + config: PQuant configuration object (or a serialized dict). + axis: Axis (or axes) the softmax normalizes over. + stable: If True, subtract the max before exponentiating for numerical stability. + input_scaler: Scalar multiplied with the logits before exponentiating. + parallelization_factor: hls4ml parallelization factor used in the ebops cost model. + quantize_input: Whether to quantize the softmax logits before the exp table. + quantize_output: Whether to quantize the exp * inv product (the softmax output). + in_quant_bits: (k, i, f) bits for the softmax input quantizer. + out_quant_bits: (k, i, f) bits for the softmax output quantizer. + exp_in_quant_bits / exp_out_quant_bits: (k, i, f) bits for the exp table. + inv_in_quant_bits / inv_out_quant_bits: (k, i, f) bits for the inv table. + """ + + def __init__( + self, + config, + axis: Union[int, Tuple[int, ...]] = -1, + stable: bool = True, + input_scaler: float = 1.0, + parallelization_factor: int = -1, + quantize_input: bool = True, + quantize_output: bool = False, + in_quant_bits: Tuple[T, T, T] = None, + out_quant_bits: Tuple[T, T, T] = None, + exp_in_quant_bits: Tuple[T, T, T] = None, + exp_out_quant_bits: Tuple[T, T, T] = None, + inv_in_quant_bits: Tuple[T, T, T] = None, + inv_out_quant_bits: Tuple[T, T, T] = None, + **kwargs, + ): + super().__init__(**kwargs) + if isinstance(config, dict): + from pquant.core.hyperparameter_optimization import PQConfig + + config = PQConfig.load_from_config(config) + self.config = config + + self.supports_masking = True + self._axis = tuple(axis) if isinstance(axis, (tuple, list)) else (axis,) + self.axes = self._axis + self.stable = stable + self.input_scaler = input_scaler + self.parallelization_factor = parallelization_factor + self.quantize_input = quantize_input + self.quantize_output = quantize_output + self.epsilon = keras.backend.epsilon() + + if in_quant_bits is not None: + self.k_input, self.i_input, self.f_input = in_quant_bits + else: + self.k_input = config.quantization_parameters.default_data_keep_negatives + self.i_input = config.quantization_parameters.default_data_integer_bits + self.f_input = config.quantization_parameters.default_data_fractional_bits + + if out_quant_bits is not None: + self.k_output, self.i_output, self.f_output = out_quant_bits + else: + self.k_output = 0 + self.i_output = config.quantization_parameters.default_data_integer_bits + self.f_output = config.quantization_parameters.default_data_fractional_bits + + self.in_quant_bits = in_quant_bits + self.out_quant_bits = out_quant_bits + self.exp_in_quant_bits = exp_in_quant_bits + self.exp_out_quant_bits = exp_out_quant_bits + self.inv_in_quant_bits = inv_in_quant_bits + self.inv_out_quant_bits = inv_out_quant_bits + + self.overflow_mode_data = config.quantization_parameters.overflow_mode_data + self.round_mode = config.quantization_parameters.round_mode + self.use_hgq = config.quantization_parameters.use_high_granularity_quantization + self.hgq_gamma = config.quantization_parameters.hgq_gamma + self.hgq_beta = config.quantization_parameters.hgq_beta + self.enable_quantization = config.quantization_parameters.enable_quantization + self.dynamic_data = config.quantization_parameters.dynamic_data_quantization + self.is_pretraining = True + + i_data = config.quantization_parameters.default_data_integer_bits + f_data = config.quantization_parameters.default_data_fractional_bits + k_data = config.quantization_parameters.default_data_keep_negatives + exp_in_quant_bits = exp_in_quant_bits if exp_in_quant_bits is not None else (k_data, i_data, f_data) + exp_out_quant_bits = exp_out_quant_bits if exp_out_quant_bits is not None else (0, i_data, f_data) + inv_in_quant_bits = inv_in_quant_bits if inv_in_quant_bits is not None else (k_data, i_data, f_data) + inv_out_quant_bits = inv_out_quant_bits if inv_out_quant_bits is not None else (0, i_data, f_data) + + def _exp(x): + if self.stable: + return ops.exp(-x * self.input_scaler) + return ops.exp(x * self.input_scaler) + + def _inv(x): + return 1.0 / (x + self.epsilon) + + self.exp_table = PQActivation( + config, + _exp, + in_quant_bits=exp_in_quant_bits, + out_quant_bits=exp_out_quant_bits, + quantize_input=stable, + quantize_output=True, + enable_ebops=stable, + ) + self.inv_table = PQActivation( + config, + _inv, + in_quant_bits=inv_in_quant_bits, + out_quant_bits=inv_out_quant_bits, + quantize_input=True, + quantize_output=True, + ) + + def build(self, input_shape): + ndim = len(input_shape) + self.axes = tuple(sorted(a if a >= 0 else a + ndim for a in self._axis)) + self.input_shape = (1,) + tuple(input_shape[1:]) + + def _data_quantizer(k, i, f): + return Quantizer( + k=k, + i=i, + f=f, + overflow=self.overflow_mode_data, + round_mode=self.round_mode, + is_data=True, + is_heterogeneous=self.use_hgq, + hgq_gamma=self.hgq_gamma, + place="datalane", + dynamic_data=self.dynamic_data, + ) + + self.input_quantizer = _data_quantizer(self.k_input, self.i_input, self.f_input) + self.output_quantizer = _data_quantizer(self.k_output, self.i_output, self.f_output) + if self.use_hgq: + self.input_quantizer.build(input_shape) + self.output_quantizer.build(input_shape) + super().build(input_shape) + + def get_input_quantization_bits(self): + return self.input_quantizer.get_quantization_bits() + + def get_output_quantization_bits(self): + return self.output_quantizer.get_quantization_bits() + + def post_pre_train_function(self): + self.is_pretraining = False + if self.quantize_input: + self.input_quantizer.post_pre_train_function() + if self.quantize_output: + self.output_quantizer.post_pre_train_function() + self.exp_table.post_pre_train_function() + self.inv_table.post_pre_train_function() + + def ebops(self): + shape = self.input_shape + accum_shape = tuple(1 if i in self.axes else s for i, s in enumerate(shape)) + max_instance = prod(accum_shape) + n_instance = self.parallelization_factor if self.parallelization_factor > 0 else max_instance + factor = n_instance / max_instance + + inp_bits = self.input_quantizer.get_total_bits(shape) + exp_bits = self.exp_table.output_quantizer.get_total_bits(shape) + inv_bits = self.inv_table.output_quantizer.get_total_bits(accum_shape) + + substract_ebops = ops.sum(inp_bits) if self.stable else 0.0 + accum_ebops = ops.sum(exp_bits) - ops.sum(ops.min(exp_bits, axis=self.axes)) + mult_ebops = ops.sum(exp_bits * inv_bits) + + ebops = substract_ebops + accum_ebops + mult_ebops + if not self.stable: + ebops = ebops + ops.sum((2.0**inp_bits) * exp_bits) * 1e-4 + ebops = ebops * factor + return ebops + self.exp_table.ebops() + self.inv_table.ebops() + + def hgq_loss(self): + if self.is_pretraining or not self.use_hgq: + return 0.0 + loss = self.hgq_beta * self.ebops() + if self.quantize_input: + loss += self.input_quantizer.hgq_loss() + if self.quantize_output: + loss += self.output_quantizer.hgq_loss() + return loss + + def call(self, inputs, training=None, mask=None): + if self.quantize_input and self.enable_quantization: + inputs = self.input_quantizer(inputs, training=training) + + if self.stable: + inputs = ops.max(inputs, axis=self.axes, keepdims=True) - inputs + + exp_inp = self.exp_table(inputs) + + if mask is not None: + exp_inp = ops.cast(mask, exp_inp.dtype) * exp_inp + + sums = ops.sum(exp_inp, axis=self.axes, keepdims=True) + divisor = self.inv_table(sums) + + out = exp_inp * divisor + if self.quantize_output and self.enable_quantization: + out = self.output_quantizer(out, training=training) + return out + + def compute_output_shape(self, input_shape): + return input_shape + + def get_config(self): + config = super().get_config() + config.update( + { + "config": self.config.get_dict(), + "axis": self.axes, + "stable": self.stable, + "input_scaler": self.input_scaler, + "parallelization_factor": self.parallelization_factor, + "quantize_input": self.quantize_input, + "quantize_output": self.quantize_output, + "in_quant_bits": self.in_quant_bits, + "out_quant_bits": self.out_quant_bits, + "exp_in_quant_bits": self.exp_in_quant_bits, + "exp_out_quant_bits": self.exp_out_quant_bits, + "inv_in_quant_bits": self.inv_in_quant_bits, + "inv_out_quant_bits": self.inv_out_quant_bits, + } + ) + return config + + @classmethod + def from_config(cls, config): + config = config.copy() + config.pop("exp_table", None) + config.pop("inv_table", None) + config.pop("input_quantizer", None) + config.pop("output_quantizer", None) + return cls(**config) diff --git a/src/pquant/core/keras/convert_to_onnx.py b/src/pquant/core/keras/convert_to_onnx.py index c35e9f4..35263c2 100644 --- a/src/pquant/core/keras/convert_to_onnx.py +++ b/src/pquant/core/keras/convert_to_onnx.py @@ -873,11 +873,9 @@ def _split_heads(x_name, pfx): k_h = _split_heads(k_proj_out, f"{prefix}_k") v_h = _split_heads(v_proj_out, f"{prefix}_v") - # --- k^T: (B, H, S, head_dim) β†’ (B, H, head_dim, S) --- k_t_name = f"{prefix}_k_T" nodes.append(oh.make_node("Transpose", inputs=[k_h], outputs=[k_t_name], perm=[0, 1, 3, 2])) - # --- Scaled dot-product scores: (B, H, T, head_dim) @ (B, H, head_dim, S) β†’ (B, H, T, S) --- raw_scores = f"{prefix}_scores_raw" scaled_scores = f"{prefix}_scores_scaled" scale_cst = f"{prefix}_attn_scale" @@ -886,13 +884,8 @@ def _split_heads(x_name, pfx): nodes.append(oh.make_node("Mul", inputs=[raw_scores, scale_cst], outputs=[scaled_scores])) current = scaled_scores - # --- Optional attn-score quantization --- - if ( - getattr(layer, "quantize_attn_scores", False) - and hasattr(layer, "attn_score_quantizer") - and getattr(layer, "enable_quantization", True) - ): - q = layer.attn_score_quantizer + if layer.softmax.quantize_input and getattr(layer, "enable_quantization", True): + q = layer.softmax.input_quantizer k_q, i_q, f_q = q.get_quantization_bits() q_nodes, current = quant_fn( f"{prefix}_attn_score_q", @@ -911,13 +904,9 @@ def _split_heads(x_name, pfx): nodes.append(oh.make_node("Softmax", inputs=[current], outputs=[attn_w_name], axis=-1)) current = attn_w_name - # --- Optional attn-weight quantization --- - if ( - getattr(layer, "quantize_attn_weights", False) - and hasattr(layer, "attn_weight_quantizer") - and getattr(layer, "enable_quantization", True) - ): - q = layer.attn_weight_quantizer + # --- Softmax output quantization (the MHA enables the softmax's output quantizer) --- + if layer.softmax.quantize_output and getattr(layer, "enable_quantization", True): + q = layer.softmax.output_quantizer k_q, i_q, f_q = q.get_quantization_bits() q_nodes, current = quant_fn( f"{prefix}_attn_weight_q", @@ -931,32 +920,10 @@ def _split_heads(x_name, pfx): ) nodes.extend(q_nodes) - # --- Context: (B, H, T, S) @ (B, H, S, head_dim) β†’ (B, H, T, head_dim) --- ctx_raw = f"{prefix}_ctx_raw" nodes.append(oh.make_node("MatMul", inputs=[current, v_h], outputs=[ctx_raw])) current_ctx = ctx_raw - # --- Optional context quantization --- - if ( - getattr(layer, "quantize_context", False) - and hasattr(layer, "context_quantizer") - and getattr(layer, "enable_quantization", True) - ): - q = layer.context_quantizer - k_q, i_q, f_q = q.get_quantization_bits() - q_nodes, current_ctx = quant_fn( - f"{prefix}_context_q", - current_ctx, - q.round_mode, - _np(k_q), - _np(i_q), - _np(f_q), - initializers, - overflow_mode=getattr(q, "overflow", "SAT"), - ) - nodes.extend(q_nodes) - - # --- Merge heads: (B, H, T, head_dim) β†’ (B, T, E) using dynamic shapes --- ctx_t = f"{prefix}_ctx_t" ctx_shape = f"{prefix}_ctx_shape" ctx_b_sc = f"{prefix}_ctx_b_sc" diff --git a/src/pquant/core/keras/layers.py b/src/pquant/core/keras/layers.py index d4db6d8..22028a1 100644 --- a/src/pquant/core/keras/layers.py +++ b/src/pquant/core/keras/layers.py @@ -1,3 +1,4 @@ +from math import prod from typing import Tuple, TypeVar import keras @@ -23,7 +24,7 @@ ) from pquant.core.hyperparameter_optimization import PQConfig -from pquant.core.keras.activations import PQActivation +from pquant.core.keras.activations import PQActivation, PQSoftmax from pquant.core.keras.quantizer import Quantizer from pquant.core.keras.utils import get_pruning_layer @@ -173,7 +174,7 @@ def get_output_quantization_bits(self): def build(self, input_shape): self.input_shape = (1,) + tuple(input_shape[1:]) - self.n_parallel = ops.prod(input_shape[1:-1]) + self.n_parallel = int(prod(input_shape[1:-1])) self.parallelization_factor = self.parallelization_factor if self.parallelization_factor > 0 else self.n_parallel self.is_pretraining = self.add_weight( shape=(), @@ -1726,19 +1727,18 @@ class PQMultiheadAttention(keras.layers.Layer): bias: Whether to add bias to projection layers. kdim: Key feature dimension (defaults to embed_dim). vdim: Value feature dimension (defaults to embed_dim). - quantize_input: Whether to quantize Q/K/V projection inputs. - quantize_output: Whether to quantize projection outputs. - quantize_attn_weights: Whether to quantize attention weights after softmax. - quantize_attn_scores: Whether to quantize attention scores before softmax. - quantize_context: Whether to quantize the context vector before merging heads. + quantize_input: Whether to quantize the Q/K/V projection inputs (the MHA inputs). + quantize_output: Whether to quantize the output projection's output (the MHA output). + The q/k/v projection outputs and the out_proj input (the context) are always + quantized, mirroring HGQ's QMultiHeadAttention. approximate_softmax: Placeholder for approximate softmax (currently uses standard softmax). in_quant_bits: (k, i, f) bits for input quantization. weight_quant_bits: (k, i, f) bits for weight quantization. bias_quant_bits: (k, i, f) bits for bias quantization. out_quant_bits: (k, i, f) bits for output quantization. - attn_quant_bits: (k, i, f) bits for attention weight quantization. - attn_score_quant_bits: (k, i, f) bits for attention score quantization. - context_quant_bits: (k, i, f) bits for context quantization. + attn_quant_bits: (k, i, f) bits for the softmax output quantizer (the attention + weights). The scores and context need no dedicated quantizers: the softmax's + input quantizer and the output projection's input quantizer cover them. Call args: inputs: A tuple (query, key, value) of tensors with shape (batch, seq, features), @@ -1762,17 +1762,12 @@ def __init__( vdim: int = None, quantize_input: bool = True, quantize_output: bool = False, - quantize_attn_weights: bool = False, - quantize_attn_scores: bool = False, - quantize_context: bool = False, approximate_softmax: bool = False, in_quant_bits: Tuple[T, T, T] = None, weight_quant_bits: Tuple[T, T, T] = None, bias_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, attn_quant_bits: Tuple[T, T, T] = None, - attn_score_quant_bits: Tuple[T, T, T] = None, - context_quant_bits: Tuple[T, T, T] = None, **kwargs, ): super().__init__(**kwargs) @@ -1789,64 +1784,71 @@ def __init__( self.use_bias = bias self.kdim = kdim if kdim is not None else embed_dim self.vdim = vdim if vdim is not None else embed_dim - self.quantize_attn_weights = quantize_attn_weights - self.quantize_attn_scores = quantize_attn_scores - self.quantize_context = quantize_context self.approximate_softmax = approximate_softmax self.scale = self.head_dim**-0.5 self.enable_quantization = config.quantization_parameters.enable_quantization self.use_hgq = config.quantization_parameters.use_high_granularity_quantization + self.hgq_beta = config.quantization_parameters.hgq_beta + self.is_pretraining = True self.in_quant_bits = in_quant_bits self.weight_quant_bits = weight_quant_bits self.bias_quant_bits = bias_quant_bits self.out_quant_bits = out_quant_bits self.attn_quant_bits = attn_quant_bits - self.attn_score_quant_bits = attn_score_quant_bits - self.context_quant_bits = context_quant_bits + self.softmax = PQSoftmax(config, -1, quantize_input=True, quantize_output=True, out_quant_bits=attn_quant_bits) proj_kwargs = dict( use_bias=bias, - quantize_input=quantize_input, - quantize_output=quantize_output, in_quant_bits=in_quant_bits, weight_quant_bits=weight_quant_bits, bias_quant_bits=bias_quant_bits, out_quant_bits=out_quant_bits, ) - self.q_proj = PQDense(config, embed_dim, enable_pruning=False, **proj_kwargs) - self.k_proj = PQDense(config, embed_dim, enable_pruning=False, **proj_kwargs) - self.v_proj = PQDense(config, embed_dim, enable_pruning=False, **proj_kwargs) - self.out_proj = PQDense(config, embed_dim, **proj_kwargs) + + qkv_kwargs = dict(quantize_input=quantize_input, quantize_output=True, **proj_kwargs) + self.q_proj = PQDense(config, embed_dim, enable_pruning=False, **qkv_kwargs) + self.k_proj = PQDense(config, embed_dim, enable_pruning=False, **qkv_kwargs) + self.v_proj = PQDense(config, embed_dim, enable_pruning=False, **qkv_kwargs) + self.out_proj = PQDense(config, embed_dim, quantize_input=True, quantize_output=quantize_output, **proj_kwargs) self.attn_dropout = keras.layers.Dropout(dropout) if dropout > 0.0 else None - def _make_data_quantizer(bits): - if bits is not None: - k, i, f = bits - else: - k = config.quantization_parameters.default_data_keep_negatives - i = config.quantization_parameters.default_data_integer_bits - f = config.quantization_parameters.default_data_fractional_bits - return Quantizer( - k=ops.convert_to_tensor(k), - i=ops.convert_to_tensor(i), - f=ops.convert_to_tensor(f), - overflow=config.quantization_parameters.overflow_mode_data, - round_mode=config.quantization_parameters.round_mode, - is_heterogeneous=config.quantization_parameters.use_high_granularity_quantization, - is_data=True, - hgq_gamma=config.quantization_parameters.hgq_gamma, - place="datalane", - dynamic_data=config.quantization_parameters.dynamic_data_quantization, - ) + def post_pre_train_function(self): + self.is_pretraining = False + for proj in (self.q_proj, self.k_proj, self.v_proj, self.out_proj): + proj.post_pre_train_function() + self.softmax.post_pre_train_function() + + def _head_bits(self, proj, seq_len): + """Bitwidths of a projection's output, in per-head layout (1, H, seq, head_dim).""" + bw = proj.output_quantizer.get_total_bits((1, seq_len, self.embed_dim)) + bw = ops.reshape(bw, (1, seq_len, self.num_heads, self.head_dim)) + return ops.transpose(bw, (0, 2, 1, 3)) + + def _attention_ebops(self): + """EBOPs of the q @ k^T and attn @ v einsums (mirrors HGQ's QMultiHeadAttention._compute_ebops).""" + attn_shape = self.softmax.input_shape # (1, H, T, S), stored when the softmax was built + query_len, key_len = attn_shape[2], attn_shape[3] + bw_q = self._head_bits(self.q_proj, query_len) + bw_k = self._head_bits(self.k_proj, key_len) + bw_v = self._head_bits(self.v_proj, key_len) + + bw_attn = self.softmax.output_quantizer.get_total_bits(attn_shape) + ebops_qk = ops.einsum("bhtd,bhsd->", bw_q, bw_k) + ebops_av = ops.einsum("bhts,bhsd->", bw_attn, bw_v) + return ebops_qk + ebops_av + + def ebops(self): + ebops = self._attention_ebops() + self.softmax.ebops() + for proj in (self.q_proj, self.k_proj, self.v_proj, self.out_proj): + ebops += proj.ebops(include_mask=proj.enable_pruning) + return ebops - if quantize_attn_weights: - self.attn_weight_quantizer = _make_data_quantizer(attn_quant_bits) - if quantize_attn_scores: - self.attn_score_quantizer = _make_data_quantizer(attn_score_quant_bits) - if quantize_context: - self.context_quantizer = _make_data_quantizer(context_quant_bits) + def hgq_loss(self): + if self.is_pretraining or not self.use_hgq: + return ops.convert_to_tensor(0.0) + return ops.convert_to_tensor(self.hgq_beta * self._attention_ebops() + self.softmax.hgq_loss()) def call( self, @@ -1875,7 +1877,6 @@ def call( k = self.k_proj(key, training=training) # (B, S, E) v = self.v_proj(value, training=training) # (B, S, E) - # Reshape to (B, H, T/S, head_dim) q = ops.reshape(q, (batch_size, query_len, self.num_heads, self.head_dim)) q = ops.transpose(q, (0, 2, 1, 3)) k = ops.reshape(k, (batch_size, key_len, self.num_heads, self.head_dim)) @@ -1895,19 +1896,13 @@ def call( attn_mask = ops.reshape(attn_mask, (batch_size, self.num_heads, query_len, key_len)) attn_scores = attn_scores + ops.cast(attn_mask, attn_scores.dtype) + mask = None if key_padding_mask is not None: - # key_padding_mask: (B, S), True means ignore -> (B, 1, 1, S) - mask = ops.cast(key_padding_mask, attn_scores.dtype) - mask = ops.reshape(mask, (batch_size, 1, 1, key_len)) - attn_scores = attn_scores + mask * -1e9 + mask = ops.logical_not(ops.cast(key_padding_mask, "bool")) + mask = ops.reshape(mask, (batch_size, 1, 1, key_len)) # (B, 1, 1, S) - if self.quantize_attn_scores and self.enable_quantization: - attn_scores = self.attn_score_quantizer(attn_scores, training=training) - - attn_weights = ops.softmax(attn_scores, axis=-1) - - if self.quantize_attn_weights and self.enable_quantization: - attn_weights = self.attn_weight_quantizer(attn_weights, training=training) + # The softmax's own input/output quantizers handle the scores and the attention weights; + attn_weights = self.softmax(attn_scores, mask=mask) if self.attn_dropout is not None: attn_weights = self.attn_dropout(attn_weights, training=training) @@ -1915,21 +1910,13 @@ def call( # Weighted sum of values: (B, H, T, head_dim) out = ops.matmul(attn_weights, v) - if self.quantize_context and self.enable_quantization: - out = self.context_quantizer(out, training=training) - # Merge heads: (B, T, E) out = ops.transpose(out, (0, 2, 1, 3)) out = ops.reshape(out, (batch_size, query_len, self.embed_dim)) out = self.out_proj(out, training=training) if self.use_hgq: - if self.quantize_attn_scores: - self.add_loss(self.attn_score_quantizer.hgq_loss()) - if self.quantize_attn_weights: - self.add_loss(self.attn_weight_quantizer.hgq_loss()) - if self.quantize_context: - self.add_loss(self.context_quantizer.hgq_loss()) + self.add_loss(self.hgq_loss()) if need_weights: # Average attention weights over heads: (B, T, S) @@ -1948,18 +1935,13 @@ def get_config(self): "kdim": self.kdim, "vdim": self.vdim, "quantize_input": self.q_proj.quantize_input, - "quantize_output": self.q_proj.quantize_output, - "quantize_attn_weights": self.quantize_attn_weights, - "quantize_attn_scores": self.quantize_attn_scores, - "quantize_context": self.quantize_context, + "quantize_output": self.out_proj.quantize_output, "approximate_softmax": self.approximate_softmax, "in_quant_bits": self.in_quant_bits, "weight_quant_bits": self.weight_quant_bits, "bias_quant_bits": self.bias_quant_bits, "out_quant_bits": self.out_quant_bits, "attn_quant_bits": self.attn_quant_bits, - "attn_score_quant_bits": self.attn_score_quant_bits, - "context_quant_bits": self.context_quant_bits, } ) return config @@ -1971,9 +1953,7 @@ def from_config(cls, config): config.pop("k_proj", None) config.pop("v_proj", None) config.pop("out_proj", None) - config.pop("attn_weight_quantizer", None) - config.pop("attn_score_quantizer", None) - config.pop("context_quantizer", None) + config.pop("softmax", None) return cls(**config) @@ -2134,7 +2114,7 @@ def post_pretrain_functions(model, config): elif isinstance(layer, PQSeparableConv2d): layer.depthwise_conv.post_pre_train_function() layer.pointwise_conv.post_pre_train_function() - elif isinstance(layer, (PQActivation, PQAvgPoolBase, PQBatchNormalization)): + elif isinstance(layer, (PQActivation, PQAvgPoolBase, PQBatchNormalization, PQSoftmax, PQMultiheadAttention)): layer.post_pre_train_function() if config.pruning_parameters.pruning_method == "pdp" or ( config.pruning_parameters.pruning_method == "wanda" and config.pruning_parameters.calculate_pruning_budget @@ -2298,7 +2278,7 @@ def get_model_losses(model, losses): loss += layer.depthwise_conv.hgq_loss() loss += layer.pointwise_conv.hgq_loss() losses += loss - elif isinstance(layer, (PQActivation, PQAvgPoolBase, PQBatchNormalization)): + elif isinstance(layer, (PQActivation, PQAvgPoolBase, PQBatchNormalization, PQSoftmax)): if layer.enable_quantization and layer.use_hgq: losses += layer.hgq_loss() return losses @@ -2831,6 +2811,6 @@ def get_ebops(model, **kwargs): for m in model.layers: if isinstance(m, (PQWeightBiasBase)): ebops += m.ebops(include_mask=m.enable_pruning) - elif isinstance(m, (PQAvgPoolBase, PQBatchNormalization, PQActivation)): + elif isinstance(m, (PQAvgPoolBase, PQBatchNormalization, PQActivation, PQSoftmax, PQMultiheadAttention)): ebops += m.ebops() return ebops diff --git a/src/pquant/core/keras/quantizer.py b/src/pquant/core/keras/quantizer.py index aa11d54..8604621 100644 --- a/src/pquant/core/keras/quantizer.py +++ b/src/pquant/core/keras/quantizer.py @@ -46,6 +46,7 @@ def calculate_bits_from_abs(self, abs_x): m = ops.ceil(ops.log(abs_x + 1e-6) / ops.log(2.0)) int_bits = ops.maximum(m, 0.0) b = self.b if hasattr(self, "b") else self.b_init + int_bits = ops.minimum(m, b - self.k) frac_bits = ops.maximum(b - int_bits - self.k_init, 0.0) return int_bits, frac_bits diff --git a/src/pquant/core/torch/activations.py b/src/pquant/core/torch/activations.py index 63e9b5e..61347ea 100644 --- a/src/pquant/core/torch/activations.py +++ b/src/pquant/core/torch/activations.py @@ -1,4 +1,5 @@ -from typing import Tuple, TypeVar +from math import prod +from typing import Tuple, TypeVar, Union import torch import torch.nn as nn @@ -40,6 +41,7 @@ def __init__( out_quant_bits: Tuple[T, T, T] = None, quantize_input=True, quantize_output=False, + enable_ebops=True, ): super().__init__() if isinstance(config, dict): @@ -61,8 +63,13 @@ def __init__( else: self.k_output, self.i_output, self.f_output = out_quant_bits - self.activation_name = activation.lower() - self.activation_function = activation_registry.get(self.activation_name) + if isinstance(activation, str): + self.activation_name = activation.lower() + self.activation_function = activation_registry.get(self.activation_name) + else: + # An activation function/callable was passed directly instead of a registry key. + self.activation_function = activation + self.activation_name = getattr(activation, "__name__", activation.__class__.__name__).lower() self.enable_quantization = config.quantization_parameters.enable_quantization self.use_hgq = config.quantization_parameters.use_high_granularity_quantization @@ -81,6 +88,7 @@ def __init__( self.saved_inputs = [] self.quantize_input = quantize_input self.quantize_output = quantize_output + self.enable_ebops = enable_ebops self.built = False def check_is_built(self, input_shape): @@ -135,6 +143,8 @@ def post_pre_train_function(self): self.is_pretraining = False def ebops(self): + if not self.enable_ebops: + return torch.tensor(0.0) bw_inp = self.input_quantizer.get_total_bits(self.input_shape) bw_out = self.output_quantizer.get_total_bits(self.input_shape) return torch.sum((2.0**bw_inp) * bw_out) * 1e-4 # type: ignore @@ -190,3 +200,205 @@ def get_config(self): def extra_repr(self): return f"quantize_input = {self.quantize_input}, quantize_output = {self.quantize_output}" + + +class PQSoftmax(nn.Module): + """Quantized softmax that mirrors HGQ's ``QSoftmax``. + + Args: + config: PQuant configuration object (or a serialized dict). + axis: Axis (or axes) the softmax normalizes over. + stable: If True, subtract the max before exponentiating for numerical stability. + input_scaler: Scalar multiplied with the logits before exponentiating. + parallelization_factor: hls4ml parallelization factor used in the ebops cost model. + quantize_input: Whether to quantize the softmax logits before the exp table. + quantize_output: Whether to quantize the exp * inv product (the softmax output). + in_quant_bits: (k, i, f) bits for the softmax input quantizer. + out_quant_bits: (k, i, f) bits for the softmax output quantizer. + exp_in_quant_bits / exp_out_quant_bits: (k, i, f) bits for the exp table. + inv_in_quant_bits / inv_out_quant_bits: (k, i, f) bits for the inv table. + """ + + def __init__( + self, + config, + axis: Union[int, Tuple[int, ...]] = -1, + stable: bool = True, + input_scaler: float = 1.0, + parallelization_factor: int = -1, + quantize_input: bool = True, + quantize_output: bool = False, + in_quant_bits: Tuple[T, T, T] = None, + out_quant_bits: Tuple[T, T, T] = None, + exp_in_quant_bits: Tuple[T, T, T] = None, + exp_out_quant_bits: Tuple[T, T, T] = None, + inv_in_quant_bits: Tuple[T, T, T] = None, + inv_out_quant_bits: Tuple[T, T, T] = None, + **kwargs, + ): + super().__init__(**kwargs) + if isinstance(config, dict): + from pquant.core.hyperparameter_optimization import PQConfig + + config = PQConfig.load_from_config(config) + self.config = config + + self._axis = tuple(axis) if isinstance(axis, (tuple, list)) else (axis,) + self.axes = self._axis + self.stable = stable + self.input_scaler = input_scaler + self.parallelization_factor = parallelization_factor + self.quantize_input = quantize_input + self.quantize_output = quantize_output + self.epsilon = 1e-7 + + if in_quant_bits is not None: + self.k_input, self.i_input, self.f_input = in_quant_bits + else: + self.k_input = config.quantization_parameters.default_data_keep_negatives + self.i_input = config.quantization_parameters.default_data_integer_bits + self.f_input = config.quantization_parameters.default_data_fractional_bits + + if out_quant_bits is not None: + self.k_output, self.i_output, self.f_output = out_quant_bits + else: + self.k_output = 0 + self.i_output = config.quantization_parameters.default_data_integer_bits + self.f_output = config.quantization_parameters.default_data_fractional_bits + + self.overflow_mode_data = config.quantization_parameters.overflow_mode_data + self.round_mode = config.quantization_parameters.round_mode + self.use_hgq = config.quantization_parameters.use_high_granularity_quantization + self.hgq_gamma = config.quantization_parameters.hgq_gamma + self.hgq_beta = config.quantization_parameters.hgq_beta + self.enable_quantization = config.quantization_parameters.enable_quantization + self.is_pretraining = True + self.built = False + + i_data = config.quantization_parameters.default_data_integer_bits + f_data = config.quantization_parameters.default_data_fractional_bits + k_data = config.quantization_parameters.default_data_keep_negatives + exp_in_quant_bits = exp_in_quant_bits if exp_in_quant_bits is not None else (k_data, i_data, f_data) + exp_out_quant_bits = exp_out_quant_bits if exp_out_quant_bits is not None else (0, i_data, f_data) + inv_in_quant_bits = inv_in_quant_bits if inv_in_quant_bits is not None else (k_data, i_data, f_data) + inv_out_quant_bits = inv_out_quant_bits if inv_out_quant_bits is not None else (0, i_data, f_data) + + def _exp(x): + if self.stable: + return torch.exp(-x * self.input_scaler) + return torch.exp(x * self.input_scaler) + + def _inv(x): + return 1.0 / (x + self.epsilon) + + self.exp_table = PQActivation( + config, + _exp, + in_quant_bits=exp_in_quant_bits, + out_quant_bits=exp_out_quant_bits, + quantize_input=stable, + quantize_output=True, + enable_ebops=stable, + ) + self.inv_table = PQActivation( + config, + _inv, + in_quant_bits=inv_in_quant_bits, + out_quant_bits=inv_out_quant_bits, + quantize_input=True, + quantize_output=True, + ) + + def check_is_built(self, input_shape): + if self.built: + return + self.built = True + ndim = len(input_shape) + self.axes = tuple(sorted(a if a >= 0 else a + ndim for a in self._axis)) + self.input_shape = (1,) + tuple(input_shape[1:]) + + def _data_quantizer(k, i, f): + return Quantizer( + k=torch.tensor(k), + i=torch.tensor(i), + f=torch.tensor(f), + overflow=self.overflow_mode_data, + round_mode=self.round_mode, + is_heterogeneous=self.use_hgq, + is_data=True, + hgq_gamma=self.hgq_gamma, + place="datalane", + dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, + ) + + self.input_quantizer = _data_quantizer(self.k_input, self.i_input, self.f_input) + self.output_quantizer = _data_quantizer(self.k_output, self.i_output, self.f_output) + if self.use_hgq: + self.input_quantizer.quantizer.build(input_shape) + self.output_quantizer.quantizer.build(input_shape) + + def get_input_quantization_bits(self): + return self.input_quantizer.get_quantization_bits() + + def get_output_quantization_bits(self): + return self.output_quantizer.get_quantization_bits() + + def post_pre_train_function(self): + self.is_pretraining = False + + def ebops(self): + shape = self.input_shape + accum_shape = tuple(1 if i in self.axes else s for i, s in enumerate(shape)) + max_instance = prod(accum_shape) + n_instance = self.parallelization_factor if self.parallelization_factor > 0 else max_instance + factor = n_instance / max_instance + + inp_bits = self.input_quantizer.get_total_bits(shape) + exp_bits = self.exp_table.output_quantizer.get_total_bits(shape) + inv_bits = self.inv_table.output_quantizer.get_total_bits(accum_shape) + + substract_ebops = torch.sum(inp_bits) if self.stable else 0.0 + accum_ebops = torch.sum(exp_bits) - torch.sum(torch.amin(exp_bits, dim=self.axes)) + mult_ebops = torch.sum(exp_bits * inv_bits) + + ebops = substract_ebops + accum_ebops + mult_ebops + if not self.stable: + ebops = ebops + torch.sum((2.0**inp_bits) * exp_bits) * 1e-4 + return ebops * factor + + def hgq_loss(self): + if self.is_pretraining or not self.use_hgq: + return torch.tensor(0.0) + loss = self.hgq_beta * self.ebops() + if self.quantize_input: + loss += self.input_quantizer.hgq_loss() + if self.quantize_output: + loss += self.output_quantizer.hgq_loss() + return loss + + def forward(self, inputs, mask=None): + self.check_is_built(inputs.shape) + if self.quantize_input and self.enable_quantization: + inputs = self.input_quantizer(inputs) + + if self.stable: + inputs = torch.amax(inputs, dim=self.axes, keepdim=True) - inputs + + exp_inp = self.exp_table(inputs) + + if mask is not None: + exp_inp = mask.to(exp_inp.dtype) * exp_inp + + sums = torch.sum(exp_inp, dim=self.axes, keepdim=True) + divisor = self.inv_table(sums) + + out = exp_inp * divisor + if self.quantize_output and self.enable_quantization: + out = self.output_quantizer(out) + return out + + def extra_repr(self) -> str: + return ( + f"axis={self.axes}, stable={self.stable}, input_scaler={self.input_scaler}, " + f"quantize_input={self.quantize_input}, quantize_output={self.quantize_output}" + ) diff --git a/src/pquant/core/torch/convert_to_onnx.py b/src/pquant/core/torch/convert_to_onnx.py index 01825c4..14a7156 100644 --- a/src/pquant/core/torch/convert_to_onnx.py +++ b/src/pquant/core/torch/convert_to_onnx.py @@ -906,13 +906,9 @@ def _split_heads(x_name, pfx): nodes.append(oh.make_node("Mul", inputs=[raw_scores, scale_cst], outputs=[scaled_scores])) current = scaled_scores - # --- Optional attn-score quantization --- - if ( - getattr(module, "quantize_attn_scores", False) - and hasattr(module, "attn_score_quantizer") - and getattr(module, "enable_quantization", True) - ): - q = module.attn_score_quantizer + # --- Softmax input quantization (the MHA enables the softmax's input quantizer) --- + if module.softmax.quantize_input and getattr(module, "enable_quantization", True): + q = module.softmax.input_quantizer k_q, i_q, f_q = q.get_quantization_bits() q_nodes, current = quant_fn( f"{prefix}_attn_score_q", @@ -931,13 +927,9 @@ def _split_heads(x_name, pfx): nodes.append(oh.make_node("Softmax", inputs=[current], outputs=[attn_w_name], axis=-1)) current = attn_w_name - # --- Optional attn-weight quantization --- - if ( - getattr(module, "quantize_attn_weights", False) - and hasattr(module, "attn_weight_quantizer") - and getattr(module, "enable_quantization", True) - ): - q = module.attn_weight_quantizer + # --- Softmax output quantization (the MHA enables the softmax's output quantizer) --- + if module.softmax.quantize_output and getattr(module, "enable_quantization", True): + q = module.softmax.output_quantizer k_q, i_q, f_q = q.get_quantization_bits() q_nodes, current = quant_fn( f"{prefix}_attn_weight_q", @@ -952,30 +944,11 @@ def _split_heads(x_name, pfx): nodes.extend(q_nodes) # --- Context: (B, H, T, S) @ (B, H, S, head_dim) β†’ (B, H, T, head_dim) --- + # No dedicated quantizer: out_proj's input quantizer (exported with the dense) covers it. ctx_raw = f"{prefix}_ctx_raw" nodes.append(oh.make_node("MatMul", inputs=[current, v_h], outputs=[ctx_raw])) current_ctx = ctx_raw - # --- Optional context quantization --- - if ( - getattr(module, "quantize_context", False) - and hasattr(module, "context_quantizer") - and getattr(module, "enable_quantization", True) - ): - q = module.context_quantizer - k_q, i_q, f_q = q.get_quantization_bits() - q_nodes, current_ctx = quant_fn( - f"{prefix}_context_q", - current_ctx, - q.round_mode, - k_q, - i_q, - f_q, - initializers, - overflow_mode=getattr(q, "overflow", "SAT"), - ) - nodes.extend(q_nodes) - # --- Merge heads: (B, H, T, head_dim) β†’ (B, T, E) using dynamic shapes --- ctx_t = f"{prefix}_ctx_t" # after Transpose β†’ (B, T, H, head_dim) ctx_shape = f"{prefix}_ctx_shape" diff --git a/src/pquant/core/torch/layers.py b/src/pquant/core/torch/layers.py index 1a28b6f..64643fe 100644 --- a/src/pquant/core/torch/layers.py +++ b/src/pquant/core/torch/layers.py @@ -7,7 +7,7 @@ from torch.fx import symbolic_trace from torch.nn.common_types import _size_1_t, _size_2_t -from pquant.core.torch.activations import PQActivation +from pquant.core.torch.activations import PQActivation, PQSoftmax from pquant.core.torch.quantizer import Quantizer from pquant.core.torch.utils import get_pruning_layer @@ -1354,14 +1354,17 @@ class PQMultiheadAttention(nn.Module): vdim: Value feature dimension (defaults to embed_dim). batch_first: If True, input/output tensors are (batch, seq, feature). If False (default), tensors are (seq, batch, feature). - quantize_input: Whether to quantize Q/K/V projection inputs. - quantize_output: Whether to quantize projection outputs. - quantize_attn_weights: Whether to quantize attention weights after softmax. + quantize_input: Whether to quantize the Q/K/V projection inputs (the MHA inputs). + quantize_output: Whether to quantize the output projection's output (the MHA output). + The q/k/v projection outputs and the out_proj input (the context) are always + quantized, mirroring HGQ's QMultiHeadAttention. in_quant_bits: (k, i, f) bits for input quantization. weight_quant_bits: (k, i, f) bits for weight quantization. bias_quant_bits: (k, i, f) bits for bias quantization. out_quant_bits: (k, i, f) bits for output quantization. - attn_quant_bits: (k, i, f) bits for attention weight quantization. + attn_quant_bits: (k, i, f) bits for the softmax output quantizer (the attention + weights). The scores and context need no dedicated quantizers: the softmax's + input quantizer and the output projection's input quantizer cover them. """ def __init__( @@ -1376,17 +1379,12 @@ def __init__( batch_first: bool = False, quantize_input: bool = True, quantize_output: bool = False, - quantize_attn_weights: bool = False, - quantize_attn_scores: bool = False, - quantize_context: bool = False, approximate_softmax: bool = False, in_quant_bits: Tuple[T, T, T] = None, weight_quant_bits: Tuple[T, T, T] = None, bias_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, attn_quant_bits: Tuple[T, T, T] = None, - attn_score_quant_bits: Tuple[T, T, T] = None, - context_quant_bits: Tuple[T, T, T] = None, **kwargs, ): super().__init__(**kwargs) @@ -1397,59 +1395,65 @@ def __init__( self.head_dim = embed_dim // num_heads self.batch_first = batch_first self.dropout = dropout - self.quantize_attn_weights = quantize_attn_weights - self.quantize_attn_scores = quantize_attn_scores - self.quantize_context = quantize_context self.approximate_softmax = approximate_softmax - self.scale = self.head_dim**-0.5 - self.softmax = nn.Softmax(dim=-1) + self.scale = float(torch.tensor(self.head_dim**-0.5, dtype=torch.float32).item()) + self.softmax = PQSoftmax(config, -1, quantize_input=True, quantize_output=True, out_quant_bits=attn_quant_bits) kdim = kdim if kdim is not None else embed_dim vdim = vdim if vdim is not None else embed_dim proj_kwargs = dict( bias=bias, - quantize_input=quantize_input, - quantize_output=quantize_output, in_quant_bits=in_quant_bits, weight_quant_bits=weight_quant_bits, bias_quant_bits=bias_quant_bits, out_quant_bits=out_quant_bits, ) - self.q_proj = PQDense(config, embed_dim, embed_dim, enable_pruning=False, **proj_kwargs) - self.k_proj = PQDense(config, kdim, embed_dim, enable_pruning=False, **proj_kwargs) - self.v_proj = PQDense(config, vdim, embed_dim, enable_pruning=False, **proj_kwargs) - self.out_proj = PQDense(config, embed_dim, embed_dim, **proj_kwargs) + qkv_kwargs = dict(quantize_input=quantize_input, quantize_output=True, **proj_kwargs) + self.q_proj = PQDense(config, embed_dim, embed_dim, enable_pruning=False, **qkv_kwargs) + self.k_proj = PQDense(config, kdim, embed_dim, enable_pruning=False, **qkv_kwargs) + self.v_proj = PQDense(config, vdim, embed_dim, enable_pruning=False, **qkv_kwargs) + self.out_proj = PQDense( + config, embed_dim, embed_dim, quantize_input=True, quantize_output=quantize_output, **proj_kwargs + ) self.attn_dropout = None + self.enable_quantization = config.quantization_parameters.enable_quantization + self.use_hgq = config.quantization_parameters.use_high_granularity_quantization + self.hgq_beta = config.quantization_parameters.hgq_beta + self.is_pretraining = True - def _make_data_quantizer(bits): - if bits is not None: - k, i, f = bits - else: - k = config.quantization_parameters.default_data_keep_negatives - i = config.quantization_parameters.default_data_integer_bits - f = config.quantization_parameters.default_data_fractional_bits - return Quantizer( - k=torch.tensor(k), - i=torch.tensor(i), - f=torch.tensor(f), - overflow=config.quantization_parameters.overflow_mode_data, - round_mode=config.quantization_parameters.round_mode, - is_heterogeneous=config.quantization_parameters.use_high_granularity_quantization, - is_data=True, - hgq_gamma=config.quantization_parameters.hgq_gamma, - place="datalane", - dynamic_data=config.quantization_parameters.dynamic_data_quantization, - ) + def post_pre_train_function(self): + # The projections, softmax and data quantizers are handled separately by the + # recursive modules() walk in post_pretrain_functions. + self.is_pretraining = False - if quantize_attn_weights: - self.attn_weight_quantizer = _make_data_quantizer(attn_quant_bits) - if quantize_attn_scores: - self.attn_score_quantizer = _make_data_quantizer(attn_score_quant_bits) - if quantize_context: - self.context_quantizer = _make_data_quantizer(context_quant_bits) - self.enable_quantization = config.quantization_parameters.enable_quantization + def _head_bits(self, proj, seq_len): + """Bitwidths of a projection's output, in per-head layout (1, H, seq, head_dim).""" + bw = proj.output_quantizer.get_total_bits((1, seq_len, self.embed_dim)) + return bw.reshape(1, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + + def _attention_ebops(self): + """EBOPs of the q @ k^T and attn @ v einsums (mirrors HGQ's QMultiHeadAttention._compute_ebops).""" + attn_shape = self.softmax.input_shape # (1, H, T, S), stored when the softmax was built + query_len, key_len = attn_shape[2], attn_shape[3] + bw_q = self._head_bits(self.q_proj, query_len) + bw_k = self._head_bits(self.k_proj, key_len) + bw_v = self._head_bits(self.v_proj, key_len) + bw_attn = self.softmax.output_quantizer.get_total_bits(attn_shape) + ebops_qk = torch.einsum("bhtd,bhsd->", bw_q, bw_k) + ebops_av = torch.einsum("bhts,bhsd->", bw_attn, bw_v) + return ebops_qk + ebops_av + + def ebops(self): + # Only the attention einsum costs are this module's own: the projections, + # softmax and lookup tables are counted by get_ebops's recursive modules() walk. + return self._attention_ebops() + + def hgq_loss(self): + if self.is_pretraining or not self.use_hgq: + return torch.tensor(0.0) + return self.hgq_beta * self._attention_ebops() def forward( self, @@ -1460,60 +1464,44 @@ def forward( attn_mask: Optional[torch.Tensor] = None, need_weights: bool = True, ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - if self.batch_first: - # (B, T, E) -> keep as-is - B, T, _ = query.shape - S = key.shape[1] - else: + if not self.batch_first: # (T, B, E) -> (B, T, E) query = query.transpose(0, 1) key = key.transpose(0, 1) value = value.transpose(0, 1) - B, T, _ = query.shape - S = key.shape[1] + + B, T = query.shape[0], query.shape[1] + S = key.shape[1] q = self.q_proj(query) # (B, T, E) k = self.k_proj(key) # (B, S, E) v = self.v_proj(value) # (B, S, E) - # Reshape to (B, H, T/S, head_dim) q = q.view(B, T, self.num_heads, self.head_dim).transpose(1, 2) k = k.view(B, S, self.num_heads, self.head_dim).transpose(1, 2) v = v.view(B, S, self.num_heads, self.head_dim).transpose(1, 2) - # Scaled dot-product attention scores: (B, H, T, S) attn_scores = torch.matmul(q, k.transpose(-2, -1)) * self.scale if attn_mask is not None: if attn_mask.dim() == 2: - # (T, S) -> (1, 1, T, S), broadcast over batch and heads attn_mask = attn_mask.unsqueeze(0).unsqueeze(0) elif attn_mask.dim() == 3: - # (B*H, T, S) -> (B, H, T, S) attn_mask = attn_mask.view(B, self.num_heads, T, S) attn_scores = attn_scores + attn_mask + mask = None if key_padding_mask is not None: - # key_padding_mask: (B, S), True means ignore - attn_scores = attn_scores.masked_fill(key_padding_mask.unsqueeze(1).unsqueeze(2), float("-inf")) + mask = ~key_padding_mask.unsqueeze(1).unsqueeze(2) # (B, 1, 1, S) - if self.quantize_attn_scores and self.enable_quantization: - attn_scores = self.attn_score_quantizer(attn_scores) - - attn_weights = self.softmax(attn_scores) - - if self.quantize_attn_weights and self.enable_quantization: - attn_weights = self.attn_weight_quantizer(attn_weights) + # The softmax's own input/output quantizers handle the scores and the attention weights; + attn_weights = self.softmax(attn_scores, mask=mask) if self.attn_dropout is not None and self.training: attn_weights = self.attn_dropout(attn_weights) - # Weighted sum of values: (B, H, T, head_dim) out = torch.matmul(attn_weights, v) - if self.quantize_context and self.enable_quantization: - out = self.context_quantizer(out) - # Merge heads: (B, T, E) out = out.transpose(1, 2).contiguous().view(B, T, self.embed_dim) out = self.out_proj(out) @@ -1529,10 +1517,7 @@ def forward( def extra_repr(self) -> str: return ( f"embed_dim={self.embed_dim}, num_heads={self.num_heads}, " - f"dropout={self.dropout}, batch_first={self.batch_first}, " - f"quantize_attn_scores={self.quantize_attn_scores}, " - f"quantize_attn_weights={self.quantize_attn_weights}, " - f"quantize_context={self.quantize_context}" + f"dropout={self.dropout}, batch_first={self.batch_first}" ) @@ -1976,6 +1961,8 @@ def post_pretrain_functions(model, config, train_loader=None, loss_function=None PQBatchNorm1d, PQLayerNorm, PQAvgPoolBase, + PQSoftmax, + PQMultiheadAttention, Quantizer, ), ): @@ -2007,6 +1994,8 @@ def post_pretrain_functions(model, config, train_loader=None, loss_function=None PQBatchNorm1d, PQLayerNorm, PQAvgPoolBase, + PQSoftmax, + PQMultiheadAttention, Quantizer, ), ): @@ -2076,7 +2065,19 @@ def get_model_losses(model, losses): if layer.use_hgq: loss += layer.hgq_loss() losses += loss - elif isinstance(layer, (PQAvgPool1d, PQAvgPool2d, PQBatchNorm2d, PQBatchNorm1d, PQLayerNorm, PQActivation)): + elif isinstance( + layer, + ( + PQAvgPool1d, + PQAvgPool2d, + PQBatchNorm2d, + PQBatchNorm1d, + PQLayerNorm, + PQActivation, + PQSoftmax, + PQMultiheadAttention, + ), + ): if layer.use_hgq: losses += layer.hgq_loss() return losses @@ -2227,7 +2228,9 @@ def get_ebops(model, **kwargs): for m in model.modules(): if isinstance(m, (PQWeightBiasBase)): ebops += m.ebops(include_mask=m.enable_pruning) - elif isinstance(m, (PQAvgPoolBase, PQBatchNorm1d, PQBatchNorm2d, PQLayerNorm, PQActivation)): + elif isinstance( + m, (PQAvgPoolBase, PQBatchNorm1d, PQBatchNorm2d, PQLayerNorm, PQActivation, PQSoftmax, PQMultiheadAttention) + ): ebops += m.ebops() return ebops diff --git a/src/pquant/core/torch/quantizer.py b/src/pquant/core/torch/quantizer.py index 1bf21cf..e9bdb1d 100644 --- a/src/pquant/core/torch/quantizer.py +++ b/src/pquant/core/torch/quantizer.py @@ -71,6 +71,7 @@ def calculate_bits_from_abs(self, abs_x): m = torch.ceil(torch.log2(abs_x + 1e-6)) int_bits = torch.clamp(m, min=0) b = self.b if hasattr(self, "b") else self.k + self.i_init + self.f_init + int_bits = torch.clamp(m, max=b - self.k.to(m.device)) frac_bits = torch.clamp(b - int_bits - self.k, min=0) return int_bits, frac_bits diff --git a/tests/test_keras_alkaid_conversion.py b/tests/test_keras_alkaid_conversion.py new file mode 100644 index 0000000..4c988fc --- /dev/null +++ b/tests/test_keras_alkaid_conversion.py @@ -0,0 +1,379 @@ +"""Convert a pruned + quantized PQuant Keras model with Alkaid""" + +import keras +import numpy as np +import pytest +from alkaid.codegen import RTLModel # noqa: E402 +from alkaid.converter import trace_model +from alkaid.trace import trace # noqa: E402 + +from pquant import pdp_config +from pquant._alkaid_plugin import _alkaid_keras_plugin # noqa: E402 +from pquant.activations import PQActivation +from pquant.core.keras.quantizer import Quantizer +from pquant.layers import ( + PQAvgPool1d, + PQAvgPool2d, + PQBatchNormalization, + PQConv1d, + PQConv2d, + PQDense, + PQDepthwiseConv2d, + PQMultiheadAttention, + PQSeparableConv2d, + PQSoftmax, + apply_final_compression, +) + +_alkaid_keras_plugin.register() + +IN_FEATURES = 3 +OUT_FEATURES = 4 +KERNEL_SIZE = 3 +H = W = 6 +SEQ_LEN = H * W + +PRUNE_FRACTION = 0.9 +INPUT_KIF = (1, 4, 4) + +IMG_SHAPE = (H, W, IN_FEATURES) +SEQ_SHAPE = (SEQ_LEN, IN_FEATURES) + + +@pytest.fixture(autouse=True) +def _channels_last(): + # Override conftest's default (channels_first); see module docstring. + keras.backend.set_image_data_format("channels_last") + + +def _build_model(config): + img_in = keras.Input(shape=IMG_SHAPE, name="img") + a = PQConv2d(config, OUT_FEATURES, KERNEL_SIZE, padding="same")(img_in) + a = PQActivation(config, activation="relu", quantize_input=True, quantize_output=True)(a) + a = keras.layers.Flatten()(a) + + seq_in = keras.Input(shape=SEQ_SHAPE, name="seq") + b = PQConv1d(config, OUT_FEATURES, KERNEL_SIZE, padding="same")(seq_in) + b = PQActivation(config, activation="relu", quantize_input=True, quantize_output=True)(b) + b = keras.layers.Flatten()(b) + + x = keras.layers.Add()([a, b]) + x = PQDense(config, units=OUT_FEATURES)(x) + x = PQActivation(config, activation="relu", quantize_input=True, quantize_output=True)(x) + return keras.Model([img_in, seq_in], x) + + +def _random_prune(layer, fraction, rng): + """Zero exactly ``fraction`` of the layer's weights via its pruning mask.""" + mask = layer.pruning_layer.mask + numel = int(np.prod(mask.shape)) + n_zero = int(round(fraction * numel)) + flat = np.ones(numel, dtype="float32") + flat[rng.permutation(numel)[:n_zero]] = 0.0 + mask.assign(flat.reshape(mask.shape)) + return n_zero / numel + + +def _build_pruned_compressed_model(config, rng): + """Build the model, build it (one forward), prune 90%, and apply final compression.""" + model = _build_model(config) + + img = np.zeros((1,) + IMG_SHAPE, dtype="float32") + seq = np.zeros((1,) + SEQ_SHAPE, dtype="float32") + + # Call once to build the quantizers and pruning masks. + model([img, seq]) + + pq_layers = [layer for layer in model.layers if isinstance(layer, (PQConv2d, PQConv1d, PQDense))] + + for layer in pq_layers: + layer._kernel.assign(rng.standard_normal(layer._kernel.shape).astype("float32")) + expected_sparsity = {layer.name: _random_prune(layer, PRUNE_FRACTION, rng) for layer in pq_layers} + + apply_final_compression(model) + return model, pq_layers, expected_sparsity + + +def test_alkaid_conversion_pruned_quantized_model(): + config = pdp_config() + config.quantization_parameters.enable_quantization = True + + rng = np.random.default_rng(0) + model, pq_layers, expected_sparsity = _build_pruned_compressed_model(config, rng) + assert {type(layer).__name__ for layer in pq_layers} == {"PQConv2d", "PQConv1d", "PQDense"} + + inp, out = trace_model(model, inputs_kif=INPUT_KIF) + + assert out.shape == (OUT_FEATURES,) + assert inp.shape == (int(np.prod(IMG_SHAPE)) + int(np.prod(SEQ_SHAPE)),) + + +def test_alkaid_rtl_matches_model(tmp_path): + config = pdp_config() + config.quantization_parameters.enable_quantization = True + + rng = np.random.default_rng(0) + model, _, _ = _build_pruned_compressed_model(config, rng) + + inp_fv, out_fv = trace_model(model, inputs_kif=INPUT_KIF) + comb = trace(inp_fv, out_fv, optimize=True) + + n_samples = 16 + img = rng.integers(0, 16, size=(n_samples,) + IMG_SHAPE).astype("float32") / 16.0 + seq = rng.integers(0, 16, size=(n_samples,) + SEQ_SHAPE).astype("float32") / 16.0 + + reference = np.asarray(model([img, seq]), dtype=np.float64) # (n_samples, OUT_FEATURES) + emulated = np.stack( + [ + np.asarray(comb(np.concatenate([img[n].ravel(), seq[n].ravel()]), quantize=True), dtype=np.float64) + for n in range(n_samples) + ] + ) + + assert np.any(reference != 0) # the comparison is non-trivial + np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) + + # Generate the actual RTL project from the same combinational logic. + RTLModel(comb, str(tmp_path), "model", flavor="verilog", print_latency=False).write() + assert (tmp_path / "src" / "model.v").exists() + + +# --- Coverage of every PQ layer the keras Alkaid plugin handles --------------- + +ALL_C = 4 +ALL_H = ALL_W = 8 +ALL_LIN = (ALL_H // 2) * (ALL_W // 2) * 2 + +ALL_IMG_SHAPE = (ALL_H, ALL_W, IN_FEATURES) +ALL_SEQ_SHAPE = (ALL_LIN, IN_FEATURES) + +ALL_KERAS_LAYER_TYPES = { + "PQConv2d", + "PQBatchNormalization", + "PQDepthwiseConv2d", + "PQSeparableConv2d", + "PQAvgPool2d", + "PQConv1d", + "PQAvgPool1d", + "PQDense", + "PQActivation", +} + + +def _build_all_layers_model(config): + """Model exercising every PQ layer type the keras Alkaid plugin handles.""" + img_in = keras.Input(shape=ALL_IMG_SHAPE, name="img") + a = PQConv2d(config, ALL_C, KERNEL_SIZE, padding="same")(img_in) + a = PQBatchNormalization(config, axis=-1)(a) + a = PQActivation(config, activation="relu", quantize_input=True, quantize_output=True)(a) + a = PQDepthwiseConv2d(config, KERNEL_SIZE, padding="same")(a) + a = PQSeparableConv2d(config, ALL_C, KERNEL_SIZE, padding="same")(a) + a = PQAvgPool2d(config, pool_size=2, strides=2, padding="valid")(a) + a = keras.layers.Flatten()(a) + + seq_in = keras.Input(shape=ALL_SEQ_SHAPE, name="seq") + b = PQConv1d(config, ALL_C, KERNEL_SIZE, padding="same")(seq_in) + b = PQActivation(config, activation="relu", quantize_input=True, quantize_output=True)(b) + b = PQAvgPool1d(config, pool_size=2, strides=2, padding="valid")(b) + b = keras.layers.Flatten()(b) + + x = keras.layers.Add()([a, b]) + x = PQDense(config, units=OUT_FEATURES)(x) + x = PQActivation(config, activation="relu", quantize_input=True, quantize_output=True)(x) + return keras.Model([img_in, seq_in], x) + + +def _all_prunable_layers(model): + """Every layer with a pruning mask, descending into PQSeparableConv2d's sub-convs.""" + found = [] + + def visit(layer): + if getattr(layer, "pruning_layer", None) is not None: + found.append(layer) + for name in ("depthwise_conv", "pointwise_conv"): + sub = getattr(layer, name, None) + if sub is not None: + visit(sub) + + for layer in model.layers: + visit(layer) + return found + + +def test_alkaid_conversion_all_layer_types(tmp_path): + config = pdp_config() + config.quantization_parameters.enable_quantization = True + + model = _build_all_layers_model(config) + rng = np.random.default_rng(0) + + # Build with random input so batchnorm running stats are sane. + model( + [ + rng.standard_normal((4,) + ALL_IMG_SHAPE).astype("float32"), + rng.standard_normal((4,) + ALL_SEQ_SHAPE).astype("float32"), + ] + ) + + assert ALL_KERAS_LAYER_TYPES <= {type(layer).__name__ for layer in model.layers} + + for layer in _all_prunable_layers(model): + layer._kernel.assign(rng.standard_normal(layer._kernel.shape).astype("float32")) + _random_prune(layer, PRUNE_FRACTION, rng) + + apply_final_compression(model) + + inp_fv, out_fv = trace_model(model, inputs_kif=INPUT_KIF) + comb = trace(inp_fv, out_fv, optimize=True) + assert out_fv.shape == (OUT_FEATURES,) + + n_samples = 16 + img = rng.integers(0, 16, size=(n_samples,) + ALL_IMG_SHAPE).astype("float32") / 16.0 + seq = rng.integers(0, 16, size=(n_samples,) + ALL_SEQ_SHAPE).astype("float32") / 16.0 + reference = np.asarray(model([img, seq]), dtype=np.float64) + emulated = np.stack( + [ + np.asarray(comb(np.concatenate([img[n].ravel(), seq[n].ravel()]), quantize=True), dtype=np.float64) + for n in range(n_samples) + ] + ) + + assert np.any(reference != 0) + np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) + + RTLModel(comb, str(tmp_path), "model", flavor="verilog", print_latency=False).write() + assert (tmp_path / "src" / "model.v").exists() + + +# --- Per-layer conversion: a model that is a single layer --------------------- + + +def _data_quantizer(config): + """A data Quantizer built from the config's default data settings.""" + qp = config.quantization_parameters + return Quantizer( + k=qp.default_data_keep_negatives, + i=qp.default_data_integer_bits, + f=qp.default_data_fractional_bits, + overflow=qp.overflow_mode_data, + round_mode=qp.round_mode, + is_heterogeneous=qp.use_high_granularity_quantization, + is_data=True, + hgq_gamma=qp.hgq_gamma, + place="datalane", + dynamic_data=qp.dynamic_data_quantization, + ) + + +def _single_layer_model(input_shape, layer, tail=None): + """A keras model that is one PQ layer, optionally followed by a Quantizer.""" + inp = keras.Input(shape=input_shape) + x = layer(inp) + if tail is not None: + x = tail(x) + return keras.Model(inp, x) + + +# id -> lambda(config) -> (input shape without batch, single-layer model). +# Layers with quantize_output set it; batchnorm (which has none) gets a trailing Quantizer. +_SINGLE_LAYER_CASES = { + "conv2d": lambda c: ( + (4, 4, 2), + _single_layer_model((4, 4, 2), PQConv2d(c, 3, KERNEL_SIZE, padding="same", quantize_output=True)), + ), + "conv1d": lambda c: ( + (8, 2), + _single_layer_model((8, 2), PQConv1d(c, 3, KERNEL_SIZE, padding="same", quantize_output=True)), + ), + "dense": lambda c: ((6,), _single_layer_model((6,), PQDense(c, units=OUT_FEATURES, quantize_output=True))), + "depthwise2d": lambda c: ( + (4, 4, 3), + _single_layer_model((4, 4, 3), PQDepthwiseConv2d(c, KERNEL_SIZE, padding="same", quantize_output=True)), + ), + "separable2d": lambda c: ( + (4, 4, 2), + _single_layer_model((4, 4, 2), PQSeparableConv2d(c, 3, KERNEL_SIZE, padding="same", quantize_output=True)), + ), + "batchnorm": lambda c: ((6,), _single_layer_model((6,), PQBatchNormalization(c, axis=-1), _data_quantizer(c))), + "avgpool2d": lambda c: ( + (4, 4, 3), + _single_layer_model((4, 4, 3), PQAvgPool2d(c, pool_size=2, strides=2, quantize_output=True)), + ), + "avgpool1d": lambda c: ( + (8, 3), + _single_layer_model((8, 3), PQAvgPool1d(c, pool_size=2, strides=2, quantize_output=True)), + ), + "activation": lambda c: ( + (6,), + _single_layer_model((6,), PQActivation(c, activation="relu", quantize_input=True, quantize_output=True)), + ), + "quantizer": lambda c: ((6,), _single_layer_model((6,), _data_quantizer(c))), + "softmax": lambda c: ((6,), _single_layer_model((6,), PQSoftmax(c, axis=-1))), +} + + +@pytest.mark.parametrize("case_id", list(_SINGLE_LAYER_CASES)) +def test_alkaid_single_layer(case_id): + config = pdp_config() + config.quantization_parameters.enable_quantization = True + input_shape, model = _SINGLE_LAYER_CASES[case_id](config) + rng = np.random.default_rng(0) + + model(rng.standard_normal((4,) + input_shape).astype("float32")) # build + apply_final_compression(model) + + inp_fv, out_fv = trace_model(model, inputs_kif=INPUT_KIF) + comb = trace(inp_fv, out_fv, optimize=True) + + n_samples = 16 + x = rng.integers(0, 16, size=(n_samples,) + input_shape).astype("float32") / 16.0 + reference = np.asarray(model(x), dtype=np.float64).reshape(n_samples, -1) + emulated = np.stack([np.asarray(comb(x[i].ravel(), quantize=True), dtype=np.float64) for i in range(n_samples)]) + + assert np.any(reference != 0) + np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) + + +# --- Multi-head attention ------------------------------------------------------ + +MHA_SEQ_LEN = 4 +MHA_EMBED_DIM = 4 +MHA_NUM_HEADS = 2 + + +def _build_mha_model(config, rng): + """Self-attention PQMultiheadAttention model with every data quantizer enabled.""" + inp = keras.Input(shape=(MHA_SEQ_LEN, MHA_EMBED_DIM)) + out, _ = PQMultiheadAttention( + config, + embed_dim=MHA_EMBED_DIM, + num_heads=MHA_NUM_HEADS, + quantize_output=True, + )(inp) + model = keras.Model(inp, out) + model(rng.standard_normal((4, MHA_SEQ_LEN, MHA_EMBED_DIM)).astype("float32")) # build + apply_final_compression(model) + return model + + +def test_alkaid_multihead_attention(tmp_path): + config = pdp_config() + config.quantization_parameters.enable_quantization = True + + rng = np.random.default_rng(0) + model = _build_mha_model(config, rng) + + inp_fv, out_fv = trace_model(model, inputs_kif=INPUT_KIF) + comb = trace(inp_fv, out_fv, optimize=True) + assert out_fv.shape == (MHA_SEQ_LEN * MHA_EMBED_DIM,) + + n_samples = 16 + x = rng.integers(0, 16, size=(n_samples, MHA_SEQ_LEN, MHA_EMBED_DIM)).astype("float32") / 16.0 + reference = np.asarray(model(x), dtype=np.float64).reshape(n_samples, -1) + emulated = np.stack([np.asarray(comb(x[i].ravel(), quantize=True), dtype=np.float64) for i in range(n_samples)]) + + assert np.any(reference != 0) + np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) + + RTLModel(comb, str(tmp_path), "model", flavor="verilog", print_latency=False).write() + assert (tmp_path / "src" / "model.v").exists() diff --git a/tests/test_torch_alkaid_conversion.py b/tests/test_torch_alkaid_conversion.py new file mode 100644 index 0000000..d62c94d --- /dev/null +++ b/tests/test_torch_alkaid_conversion.py @@ -0,0 +1,390 @@ +"""Convert a pruned + quantized PQuant Torch model with Alkaid""" + +import numpy as np +import pytest +import torch +import torch.nn as nn +from alkaid.codegen import RTLModel # noqa: E402 +from alkaid.converter import trace_model +from alkaid.trace import trace # noqa: E402 +from alkaid.trace import FVArray, HWConfig # noqa: E402 + +from pquant import pdp_config +from pquant._alkaid_plugin import _alkaid_torch_plugin # noqa: E402 +from pquant.core.torch.activations import PQActivation +from pquant.core.torch.layers import ( + PQAvgPool1d, + PQAvgPool2d, + PQBatchNorm1d, + PQBatchNorm2d, + PQConv1d, + PQConv2d, + PQDense, + PQMultiheadAttention, + PQSoftmax, + apply_final_compression, +) +from pquant.core.torch.quantizer import Quantizer + +_alkaid_torch_plugin.register() + +IN_FEATURES = 3 +OUT_FEATURES = 4 +KERNEL_SIZE = 3 +H = W = 6 +SEQ_LEN = H * W + +PRUNE_FRACTION = 0.9 +HWCONF = HWConfig(1, 1, -1) +INPUT_KIF = (1, 4, 4) + + +class TwoBranchNet(nn.Module): + """conv2d branch + conv1d branch merged (matched flatten lengths) -> dense head. + + A conv after a reshape/flatten cannot be traced by Alkaid (its reshape folds + the batch axis), and its ``Concatenate`` merge is unreliable, so the two convs + live on separate branches that are summed before the dense head. + """ + + def __init__(self, config): + super().__init__() + self.conv2d = PQConv2d(config, IN_FEATURES, OUT_FEATURES, KERNEL_SIZE, padding="same") + self.act2d = PQActivation(config, "relu", quantize_input=True, quantize_output=True) + self.flat2d = nn.Flatten() + + self.conv1d = PQConv1d(config, IN_FEATURES, OUT_FEATURES, KERNEL_SIZE, padding="same") + self.act1d = PQActivation(config, "relu", quantize_input=True, quantize_output=True) + self.flat1d = nn.Flatten() + + self.dense = PQDense(config, OUT_FEATURES * SEQ_LEN, OUT_FEATURES) + self.act = PQActivation(config, "relu", quantize_input=True, quantize_output=True) + + def forward(self, img, seq): + a = self.flat2d(self.act2d(self.conv2d(img))) + b = self.flat1d(self.act1d(self.conv1d(seq))) + x = a + b + return self.act(self.dense(x)) + + +def _random_prune(layer, fraction, rng): + """Zero exactly ``fraction`` of the layer's weights via its pruning mask.""" + mask = layer.pruning_layer.mask + numel = int(np.prod(tuple(mask.shape))) + n_zero = int(round(fraction * numel)) + flat = np.ones(numel, dtype="float32") + flat[rng.permutation(numel)[:n_zero]] = 0.0 + mask.copy_(torch.tensor(flat.reshape(tuple(mask.shape)), dtype=mask.dtype, device=mask.device)) + return n_zero / numel + + +def _fixed_point_input(shape, kif=INPUT_KIF): + """Bounded fixed-point symbolic input so the SAT input quantizer can be replayed.""" + k, i, f = (np.full(shape, v, dtype=np.int8) for v in kif) + return FVArray.from_kif(k, i, f, HWCONF, 0, None) + + +def _build_pruned_compressed_model(config, rng): + """Build the model, build it (one forward), prune 90%, apply final compression, eval.""" + model = TwoBranchNet(config) + device = next(model.parameters()).device + img = torch.zeros(1, IN_FEATURES, H, W, device=device) + seq = torch.zeros(1, IN_FEATURES, SEQ_LEN, device=device) + + with torch.no_grad(): + model(img, seq) # build quantizers + pruning masks + + pq_layers = [m for m in model.modules() if isinstance(m, (PQConv2d, PQConv1d, PQDense))] + for layer in pq_layers: + layer._weight.copy_( + torch.tensor(rng.standard_normal(tuple(layer._weight.shape)), dtype=layer._weight.dtype, device=device) + ) + expected_sparsity = {id(layer): _random_prune(layer, PRUNE_FRACTION, rng) for layer in pq_layers} + + apply_final_compression(model) + model.eval() + return model, pq_layers, expected_sparsity, device + + +def test_alkaid_conversion_pruned_quantized_model(): + config = pdp_config() + config.quantization_parameters.enable_quantization = True + + rng = np.random.default_rng(0) + model, _, _, _ = _build_pruned_compressed_model(config, rng) + + inputs = (_fixed_point_input((1, IN_FEATURES, H, W)), _fixed_point_input((1, IN_FEATURES, SEQ_LEN))) + inp, out = trace_model(model, hwconf=HWCONF, inputs=inputs, framework="torch") + + assert out.shape == (OUT_FEATURES,) + expected_inputs = IN_FEATURES * H * W + IN_FEATURES * SEQ_LEN + assert inp.shape == (expected_inputs,) + + +def test_alkaid_rtl_matches_model(tmp_path): + config = pdp_config() + config.quantization_parameters.enable_quantization = True + + rng = np.random.default_rng(0) + model, _, _, device = _build_pruned_compressed_model(config, rng) + + inputs = (_fixed_point_input((1, IN_FEATURES, H, W)), _fixed_point_input((1, IN_FEATURES, SEQ_LEN))) + inp_fv, out_fv = trace_model(model, hwconf=HWCONF, inputs=inputs, framework="torch") + comb = trace(inp_fv, out_fv, optimize=True) + n_samples = 16 + img = rng.integers(0, 16, size=(n_samples, IN_FEATURES, H, W)).astype("float32") / 16.0 + seq = rng.integers(0, 16, size=(n_samples, IN_FEATURES, SEQ_LEN)).astype("float32") / 16.0 + + with torch.no_grad(): + reference = ( + model(torch.tensor(img, device=device), torch.tensor(seq, device=device)).cpu().numpy().astype(np.float64) + ) # (n_samples, OUT_FEATURES) + + emulated = np.stack( + [ + np.asarray(comb(np.concatenate([img[n].ravel(), seq[n].ravel()]), quantize=True), dtype=np.float64) + for n in range(n_samples) + ] + ) + + assert np.any(reference != 0) # the comparison is non-trivial + np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) + + # Generate the actual RTL project from the same combinational logic. + RTLModel(comb, str(tmp_path), "model", flavor="verilog", print_latency=False).write() + assert (tmp_path / "src" / "model.v").exists() + + +# --- Coverage of every PQ layer the torch Alkaid plugin handles --------------- + +ALL_C = 4 +ALL_H = ALL_W = 6 +ALL_LIN = (ALL_H // 2) * (ALL_W // 2) * 2 + +ALL_TORCH_LAYER_TYPES = { + "PQConv2d", + "PQBatchNorm2d", + "PQAvgPool2d", + "PQConv1d", + "PQBatchNorm1d", + "PQAvgPool1d", + "PQDense", + "PQActivation", +} + + +class AllLayersNet(nn.Module): + """Exercises every PQ layer type the torch Alkaid plugin handles. + + conv2d -> batchnorm2d -> relu -> avgpool2d branch, and a + conv1d -> batchnorm1d -> relu -> avgpool1d branch, merged (matched flatten + lengths) into a dense head. Each layer also drives an inner Quantizer. + """ + + def __init__(self, config): + super().__init__() + self.conv2d = PQConv2d(config, IN_FEATURES, ALL_C, KERNEL_SIZE, padding="same") + self.bn2d = PQBatchNorm2d(config, ALL_C) + self.act2d = PQActivation(config, "relu", quantize_input=True, quantize_output=True) + self.pool2d = PQAvgPool2d(config, kernel_size=2, stride=2) + self.flat2d = nn.Flatten() + + self.conv1d = PQConv1d(config, IN_FEATURES, ALL_C, KERNEL_SIZE, padding="same") + self.bn1d = PQBatchNorm1d(config, ALL_C) + self.act1d = PQActivation(config, "relu", quantize_input=True, quantize_output=True) + self.pool1d = PQAvgPool1d(config, kernel_size=2, stride=2) + self.flat1d = nn.Flatten() + + self.dense = PQDense(config, ALL_C * (ALL_H // 2) * (ALL_W // 2), OUT_FEATURES) + self.act = PQActivation(config, "relu", quantize_input=True, quantize_output=True) + + def forward(self, img, seq): + a = self.flat2d(self.pool2d(self.act2d(self.bn2d(self.conv2d(img))))) + b = self.flat1d(self.pool1d(self.act1d(self.bn1d(self.conv1d(seq))))) + return self.act(self.dense(a + b)) + + +def test_alkaid_conversion_all_layer_types(tmp_path): + config = pdp_config() + config.quantization_parameters.enable_quantization = True + + rng = np.random.default_rng(0) + model = AllLayersNet(config) + device = next(model.parameters()).device + + model.train() + with torch.no_grad(): + model( + torch.tensor(rng.standard_normal((4, IN_FEATURES, ALL_H, ALL_W)), dtype=torch.float32, device=device), + torch.tensor(rng.standard_normal((4, IN_FEATURES, ALL_LIN)), dtype=torch.float32, device=device), + ) + + assert ALL_TORCH_LAYER_TYPES <= {type(m).__name__ for m in model.modules()} + + with torch.no_grad(): + for layer in [m for m in model.modules() if getattr(m, "pruning_layer", None) is not None]: + layer._weight.copy_( + torch.tensor(rng.standard_normal(tuple(layer._weight.shape)), dtype=layer._weight.dtype, device=device) + ) + _random_prune(layer, PRUNE_FRACTION, rng) + + apply_final_compression(model) + model.eval() + + inputs = (_fixed_point_input((1, IN_FEATURES, ALL_H, ALL_W)), _fixed_point_input((1, IN_FEATURES, ALL_LIN))) + inp_fv, out_fv = trace_model(model, hwconf=HWCONF, inputs=inputs, framework="torch") + comb = trace(inp_fv, out_fv, optimize=True) + assert out_fv.shape == (OUT_FEATURES,) + + n_samples = 16 + img = rng.integers(0, 16, size=(n_samples, IN_FEATURES, ALL_H, ALL_W)).astype("float32") / 16.0 + seq = rng.integers(0, 16, size=(n_samples, IN_FEATURES, ALL_LIN)).astype("float32") / 16.0 + with torch.no_grad(): + reference = ( + model(torch.tensor(img, device=device), torch.tensor(seq, device=device)).cpu().numpy().astype(np.float64) + ) + emulated = np.stack( + [ + np.asarray(comb(np.concatenate([img[n].ravel(), seq[n].ravel()]), quantize=True), dtype=np.float64) + for n in range(n_samples) + ] + ) + + assert np.any(reference != 0) + np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) + + RTLModel(comb, str(tmp_path), "model", flavor="verilog", print_latency=False).write() + assert (tmp_path / "src" / "model.v").exists() + + +# --- Per-layer conversion: a model that is a single layer --------------------- + + +def _data_quantizer(config): + """A data Quantizer built from the config's default data settings.""" + qp = config.quantization_parameters + return Quantizer( + k=qp.default_data_keep_negatives, + i=qp.default_data_integer_bits, + f=qp.default_data_fractional_bits, + overflow=qp.overflow_mode_data, + round_mode=qp.round_mode, + is_heterogeneous=qp.use_high_granularity_quantization, + is_data=True, + hgq_gamma=qp.hgq_gamma, + place="datalane", + dynamic_data=qp.dynamic_data_quantization, + ) + + +class _SingleLayer(nn.Module): + """One PQ layer, optionally followed by a Quantizer. + + Layers with a ``quantize_output`` option set it directly; layers without one + get an explicit trailing Quantizer so the output is fixed-point. + """ + + def __init__(self, layer, tail=None): + super().__init__() + self.layer = layer + self.tail = tail + + def forward(self, x): + x = self.layer(x) + return x if self.tail is None else self.tail(x) + + +_SINGLE_LAYER_CASES = { + "conv2d": lambda c: ((1, 2, 4, 4), _SingleLayer(PQConv2d(c, 2, 3, KERNEL_SIZE, padding="same", quantize_output=True))), + "conv1d": lambda c: ((1, 2, 8), _SingleLayer(PQConv1d(c, 2, 3, KERNEL_SIZE, padding="same", quantize_output=True))), + "dense": lambda c: ((1, 6), _SingleLayer(PQDense(c, 6, OUT_FEATURES, quantize_output=True))), + "batchnorm2d": lambda c: ((1, 3, 4, 4), _SingleLayer(PQBatchNorm2d(c, 3), _data_quantizer(c))), + "batchnorm1d": lambda c: ((1, 3, 8), _SingleLayer(PQBatchNorm1d(c, 3), _data_quantizer(c))), + "avgpool2d": lambda c: ((1, 3, 4, 4), _SingleLayer(PQAvgPool2d(c, kernel_size=2, stride=2, quantize_output=True))), + "avgpool1d": lambda c: ((1, 3, 8), _SingleLayer(PQAvgPool1d(c, kernel_size=2, stride=2, quantize_output=True))), + "activation": lambda c: ((1, 6), _SingleLayer(PQActivation(c, "relu", quantize_input=True, quantize_output=True))), + "quantizer": lambda c: ((1, 6), _SingleLayer(_data_quantizer(c))), + "softmax": lambda c: ((1, 6), _SingleLayer(PQSoftmax(c, axis=-1))), +} + + +@pytest.mark.parametrize("case_id", list(_SINGLE_LAYER_CASES)) +def test_alkaid_single_layer(case_id): + config = pdp_config() + config.quantization_parameters.enable_quantization = True + shape, model = _SINGLE_LAYER_CASES[case_id](config) + rng = np.random.default_rng(0) + + model.train() + with torch.no_grad(): + model(torch.tensor(rng.standard_normal((4,) + shape[1:]), dtype=torch.float32)) + apply_final_compression(model) + model.eval() + + inp_fv, out_fv = trace_model(model, hwconf=HWCONF, inputs=(_fixed_point_input(shape),), framework="torch") + comb = trace(inp_fv, out_fv, optimize=True) + + n_samples = 16 + x = rng.integers(0, 16, size=(n_samples,) + shape[1:]).astype("float32") / 16.0 + with torch.no_grad(): + reference = model(torch.tensor(x)).cpu().numpy().reshape(n_samples, -1).astype(np.float64) + emulated = np.stack([np.asarray(comb(x[i].ravel(), quantize=True), dtype=np.float64) for i in range(n_samples)]) + + assert np.any(reference != 0) + np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) + + +# --- Multi-head attention ------------------------------------------------------ + +MHA_SEQ_LEN = 4 +MHA_EMBED_DIM = 4 +MHA_NUM_HEADS = 2 + + +class _MHANet(nn.Module): + """Self-attention PQMultiheadAttention with every data quantizer enabled. + + The MHA lives inside a wrapper module so torch.fx inlines its forward with + concrete (None) mask arguments; tracing the MHA as the fx root would turn the + masks into proxies and hit data-dependent control flow. + """ + + def __init__(self, config): + super().__init__() + self.mha = PQMultiheadAttention( + config, + embed_dim=MHA_EMBED_DIM, + num_heads=MHA_NUM_HEADS, + batch_first=True, + quantize_output=True, + ) + + def forward(self, x): + out, _ = self.mha(x, x, x) + return out + + +def test_alkaid_multihead_attention(tmp_path): + config = pdp_config() + config.quantization_parameters.enable_quantization = True + + rng = np.random.default_rng(0) + model = _MHANet(config) + with torch.no_grad(): + model(torch.tensor(rng.standard_normal((4, MHA_SEQ_LEN, MHA_EMBED_DIM)), dtype=torch.float32)) # build + apply_final_compression(model) + model.eval() + + shape = (1, MHA_SEQ_LEN, MHA_EMBED_DIM) + inp_fv, out_fv = trace_model(model, hwconf=HWCONF, inputs=(_fixed_point_input(shape),), framework="torch") + comb = trace(inp_fv, out_fv, optimize=True) + assert out_fv.shape == (MHA_SEQ_LEN * MHA_EMBED_DIM,) + + n_samples = 16 + x = rng.integers(0, 16, size=(n_samples, MHA_SEQ_LEN, MHA_EMBED_DIM)).astype("float32") / 16.0 + with torch.no_grad(): + reference = model(torch.tensor(x)).cpu().numpy().reshape(n_samples, -1).astype(np.float64) + emulated = np.stack([np.asarray(comb(x[i].ravel(), quantize=True), dtype=np.float64) for i in range(n_samples)]) + + assert np.any(reference != 0) + np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) From b1c3ec89b70b6e2f64317706f45a766b026d3409 Mon Sep 17 00:00:00 2001 From: nroope Date: Fri, 12 Jun 2026 14:26:02 +0200 Subject: [PATCH 09/22] per-tensor-granularity-for-hgq (#44) * Use config quantization parameter named granularity to determine HGQ granularity, either per-tensor or per-weight --- src/pquant/core/keras/activations.py | 2 + src/pquant/core/keras/layers.py | 114 +++++++++++- src/pquant/core/keras/quantizer.py | 54 +++++- src/pquant/core/torch/activations.py | 2 + src/pquant/core/torch/hgq_quantizer.py | 37 ++-- src/pquant/core/torch/layers.py | 109 ++++++++++- src/pquant/core/torch/quantizer.py | 26 ++- tests/conftest.py | 7 +- tests/run_tests.sh | 6 +- tests/test_hgq_keras.py | 241 ++++++++++++++++++++++++ tests/test_hgq_torch.py | 248 ++++++++++++++++++++++++- tests/test_keras_compression_layers.py | 2 + tests/test_torch_compression_layers.py | 4 + 13 files changed, 814 insertions(+), 38 deletions(-) create mode 100644 tests/test_hgq_keras.py diff --git a/src/pquant/core/keras/activations.py b/src/pquant/core/keras/activations.py index f837366..39a7699 100644 --- a/src/pquant/core/keras/activations.py +++ b/src/pquant/core/keras/activations.py @@ -102,6 +102,7 @@ def build(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.dynamic_data, + granularity=self.config.quantization_parameters.granularity, ) if self.quantize_output: self.output_quantizer = Quantizer( @@ -115,6 +116,7 @@ def build(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.dynamic_data, + granularity=self.config.quantization_parameters.granularity, ) if self.use_multiplier: diff --git a/src/pquant/core/keras/layers.py b/src/pquant/core/keras/layers.py index 22028a1..4a058c0 100644 --- a/src/pquant/core/keras/layers.py +++ b/src/pquant/core/keras/layers.py @@ -43,6 +43,10 @@ def __init__( weight_quant_bits: Tuple[T, T, T] = None, bias_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + weight_quant_granularity=None, + in_quant_granularity=None, + bias_quant_granularity=None, + out_quant_granularity=None, enable_pruning=None, *args, **kwargs, @@ -87,6 +91,10 @@ def __init__( self.weight_quant_bits = weight_quant_bits self.bias_quant_bits = bias_quant_bits self.out_quant_bits = out_quant_bits + self.weight_quant_granularity = weight_quant_granularity + self.in_quant_granularity = in_quant_granularity + self.bias_quant_granularity = bias_quant_granularity + self.out_quant_granularity = out_quant_granularity self.pruning_first = config.training_parameters.pruning_first self.enable_quantization = config.quantization_parameters.enable_quantization self.round_mode = config.quantization_parameters.round_mode @@ -107,6 +115,12 @@ def __init__( self._is_finetuning = False self.config = config + # Each quantizer follows the config granularity unless its per-quantizer override is set. + weight_granularity = weight_quant_granularity if weight_quant_granularity is not None else self.granularity + bias_granularity = bias_quant_granularity if bias_quant_granularity is not None else self.granularity + in_granularity = in_quant_granularity if in_quant_granularity is not None else self.granularity + out_granularity = out_quant_granularity if out_quant_granularity is not None else self.granularity + self.weight_quantizer = Quantizer( k=ops.convert_to_tensor(self.k_weight), i=ops.convert_to_tensor(self.i_weight), @@ -115,7 +129,7 @@ def __init__( round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=False, - granularity=self.granularity, + granularity=weight_granularity, hgq_gamma=self.hgq_gamma, place="weight", ) @@ -129,6 +143,7 @@ def __init__( round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=False, + granularity=bias_granularity, hgq_gamma=self.hgq_gamma, place="bias", ) @@ -140,6 +155,7 @@ def __init__( round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=True, + granularity=in_granularity, hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.dynamic_data, @@ -152,6 +168,7 @@ def __init__( round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=True, + granularity=out_granularity, hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.dynamic_data, @@ -316,6 +333,10 @@ def get_config(self): "weight_quant_bits": self.weight_quant_bits, "bias_quant_bits": self.bias_quant_bits, "out_quant_bits": self.out_quant_bits, + "weight_quant_granularity": self.weight_quant_granularity, + "in_quant_granularity": self.in_quant_granularity, + "bias_quant_granularity": self.bias_quant_granularity, + "out_quant_granularity": self.out_quant_granularity, "enable_pruning": self.enable_pruning, "final_compression_done": self.final_compression_done, } @@ -352,6 +373,10 @@ def __init__( weight_quant_bits: Tuple[T, T, T] = None, bias_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + weight_quant_granularity=None, + in_quant_granularity=None, + bias_quant_granularity=None, + out_quant_granularity=None, enable_pruning=None, **kwargs, ): @@ -379,6 +404,10 @@ def __init__( weight_quant_bits=weight_quant_bits, bias_quant_bits=bias_quant_bits, out_quant_bits=out_quant_bits, + weight_quant_granularity=weight_quant_granularity, + in_quant_granularity=in_quant_granularity, + bias_quant_granularity=bias_quant_granularity, + out_quant_granularity=out_quant_granularity, enable_pruning=enable_pruning, **kwargs, ) @@ -577,6 +606,10 @@ def __init__( weight_quant_bits: Tuple[T, T, T] = None, bias_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + weight_quant_granularity=None, + in_quant_granularity=None, + bias_quant_granularity=None, + out_quant_granularity=None, enable_pruning=None, **kwargs, ): @@ -589,6 +622,10 @@ def __init__( weight_quant_bits=weight_quant_bits, bias_quant_bits=bias_quant_bits, out_quant_bits=out_quant_bits, + weight_quant_granularity=weight_quant_granularity, + in_quant_granularity=in_quant_granularity, + bias_quant_granularity=bias_quant_granularity, + out_quant_granularity=out_quant_granularity, enable_pruning=enable_pruning, activity_regularizer=activity_regularizer, **kwargs, @@ -890,6 +927,10 @@ def __init__( weight_quant_bits: Tuple[T, T, T] = None, bias_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + weight_quant_granularity=None, + in_quant_granularity=None, + bias_quant_granularity=None, + out_quant_granularity=None, enable_pruning=None, strides=1, padding="valid", @@ -916,6 +957,10 @@ def __init__( weight_quant_bits=weight_quant_bits, bias_quant_bits=bias_quant_bits, out_quant_bits=out_quant_bits, + weight_quant_granularity=weight_quant_granularity, + in_quant_granularity=in_quant_granularity, + bias_quant_granularity=bias_quant_granularity, + out_quant_granularity=out_quant_granularity, enable_pruning=enable_pruning, activity_regularizer=activity_regularizer, **kwargs, @@ -1116,6 +1161,10 @@ def __init__( weight_quant_bits: Tuple[T, T, T] = None, bias_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + weight_quant_granularity=None, + in_quant_granularity=None, + bias_quant_granularity=None, + out_quant_granularity=None, enable_pruning=None, use_bias=True, kernel_initializer="glorot_uniform", @@ -1135,6 +1184,10 @@ def __init__( weight_quant_bits=weight_quant_bits, bias_quant_bits=bias_quant_bits, out_quant_bits=out_quant_bits, + weight_quant_granularity=weight_quant_granularity, + in_quant_granularity=in_quant_granularity, + bias_quant_granularity=bias_quant_granularity, + out_quant_granularity=out_quant_granularity, enable_pruning=enable_pruning, **kwargs, ) @@ -1278,6 +1331,9 @@ def __init__( synchronized=False, quantize_input=True, quantize_parameters=True, + in_quant_granularity=None, + weight_quant_granularity=None, + bias_quant_granularity=None, **kwargs, ): if isinstance(config, dict): @@ -1311,6 +1367,9 @@ def __init__( self.quantize_input = quantize_input self.quantize_parameters = quantize_parameters self.granularity = config.quantization_parameters.granularity + self.in_quant_granularity = in_quant_granularity + self.weight_quant_granularity = weight_quant_granularity + self.bias_quant_granularity = bias_quant_granularity self.dynamic_data = config.quantization_parameters.dynamic_data_quantization self.config = config self.f_weight = self.f_bias = ops.convert_to_tensor(config.quantization_parameters.default_weight_fractional_bits) @@ -1329,6 +1388,9 @@ def build(self, input_shape): trainable=False, dtype="float32", ) + in_granularity = self.in_quant_granularity if self.in_quant_granularity is not None else self.granularity + weight_granularity = self.weight_quant_granularity if self.weight_quant_granularity is not None else self.granularity + bias_granularity = self.bias_quant_granularity if self.bias_quant_granularity is not None else self.granularity self.input_quantizer = Quantizer( k=1.0, i=self.i_input, @@ -1340,6 +1402,7 @@ def build(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.dynamic_data, + granularity=in_granularity, ) self.weight_quantizer = Quantizer( k=1.0, @@ -1350,6 +1413,7 @@ def build(self, input_shape): is_data=False, is_heterogeneous=self.use_hgq, place="weight", + granularity=weight_granularity, ) self.bias_quantizer = Quantizer( k=1.0, @@ -1360,6 +1424,7 @@ def build(self, input_shape): is_data=False, is_heterogeneous=self.use_hgq, place="bias", + granularity=bias_granularity, ) self.input_quantizer.build(input_shape) self.weight_quantizer.build(self.moving_variance.shape) @@ -1480,6 +1545,9 @@ def get_config(self): "config": self.config.get_dict(), "quantize_input": self.quantize_input, "quantize_parameters": self.quantize_parameters, + "in_quant_granularity": self.in_quant_granularity, + "weight_quant_granularity": self.weight_quant_granularity, + "bias_quant_granularity": self.bias_quant_granularity, "final_compression_done": self.final_compression_done, } ) @@ -1495,6 +1563,8 @@ def __init__( quantize_output=False, in_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + in_quant_granularity=None, + out_quant_granularity=None, **kwargs, ): @@ -1504,6 +1574,8 @@ def __init__( self.in_quant_bits = in_quant_bits self.out_quant_bits = out_quant_bits + self.in_quant_granularity = in_quant_granularity + self.out_quant_granularity = out_quant_granularity if in_quant_bits is not None: self.k_input, self.i_input, self.f_input = in_quant_bits @@ -1548,6 +1620,9 @@ def build(self, input_shape): trainable=False, dtype="float32", ) + config_granularity = self.config.quantization_parameters.granularity + in_granularity = self.in_quant_granularity if self.in_quant_granularity is not None else config_granularity + out_granularity = self.out_quant_granularity if self.out_quant_granularity is not None else config_granularity self.input_quantizer = Quantizer( k=1.0, i=self.i_input, @@ -1559,6 +1634,7 @@ def build(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.dynamic_data, + granularity=in_granularity, ) self.output_quantizer = Quantizer( k=1.0, @@ -1571,6 +1647,7 @@ def build(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.dynamic_data, + granularity=out_granularity, ) self.input_quantizer.build(input_shape) self.output_quantizer.build(self.compute_output_shape(input_shape)) @@ -1624,6 +1701,8 @@ def get_config(self): "quantize_output": self.quantize_output, "in_quant_bits": self.in_quant_bits, "out_quant_bits": self.out_quant_bits, + "in_quant_granularity": self.in_quant_granularity, + "out_quant_granularity": self.out_quant_granularity, } ) return config @@ -1639,6 +1718,8 @@ def __init__( quantize_output=False, in_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + in_quant_granularity=None, + out_quant_granularity=None, strides=None, padding="valid", data_format=None, @@ -1656,6 +1737,8 @@ def __init__( quantize_output=quantize_output, in_quant_bits=in_quant_bits, out_quant_bits=out_quant_bits, + in_quant_granularity=in_quant_granularity, + out_quant_granularity=out_quant_granularity, **kwargs, ) @@ -1681,6 +1764,8 @@ def __init__( quantize_output=False, in_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + in_quant_granularity=None, + out_quant_granularity=None, strides=None, padding="valid", data_format=None, @@ -1698,6 +1783,8 @@ def __init__( quantize_output=quantize_output, in_quant_bits=in_quant_bits, out_quant_bits=out_quant_bits, + in_quant_granularity=in_quant_granularity, + out_quant_granularity=out_quant_granularity, ) def call(self, x, training=None): @@ -1768,6 +1855,9 @@ def __init__( bias_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, attn_quant_bits: Tuple[T, T, T] = None, + in_quant_granularity=None, + out_quant_granularity=None, + param_quant_granularity=None, **kwargs, ): super().__init__(**kwargs) @@ -1797,6 +1887,10 @@ def __init__( self.out_quant_bits = out_quant_bits self.attn_quant_bits = attn_quant_bits + self.in_quant_granularity = in_quant_granularity + self.out_quant_granularity = out_quant_granularity + self.param_quant_granularity = param_quant_granularity + self.softmax = PQSoftmax(config, -1, quantize_input=True, quantize_output=True, out_quant_bits=attn_quant_bits) proj_kwargs = dict( use_bias=bias, @@ -1804,13 +1898,24 @@ def __init__( weight_quant_bits=weight_quant_bits, bias_quant_bits=bias_quant_bits, out_quant_bits=out_quant_bits, + weight_quant_granularity=param_quant_granularity, + bias_quant_granularity=param_quant_granularity, ) - qkv_kwargs = dict(quantize_input=quantize_input, quantize_output=True, **proj_kwargs) + qkv_kwargs = dict( + quantize_input=quantize_input, quantize_output=True, in_quant_granularity=in_quant_granularity, **proj_kwargs + ) self.q_proj = PQDense(config, embed_dim, enable_pruning=False, **qkv_kwargs) self.k_proj = PQDense(config, embed_dim, enable_pruning=False, **qkv_kwargs) self.v_proj = PQDense(config, embed_dim, enable_pruning=False, **qkv_kwargs) - self.out_proj = PQDense(config, embed_dim, quantize_input=True, quantize_output=quantize_output, **proj_kwargs) + self.out_proj = PQDense( + config, + embed_dim, + quantize_input=True, + quantize_output=quantize_output, + out_quant_granularity=out_quant_granularity, + **proj_kwargs, + ) self.attn_dropout = keras.layers.Dropout(dropout) if dropout > 0.0 else None @@ -1942,6 +2047,9 @@ def get_config(self): "bias_quant_bits": self.bias_quant_bits, "out_quant_bits": self.out_quant_bits, "attn_quant_bits": self.attn_quant_bits, + "in_quant_granularity": self.in_quant_granularity, + "out_quant_granularity": self.out_quant_granularity, + "param_quant_granularity": self.param_quant_granularity, } ) return config diff --git a/src/pquant/core/keras/quantizer.py b/src/pquant/core/keras/quantizer.py index 8604621..20910aa 100644 --- a/src/pquant/core/keras/quantizer.py +++ b/src/pquant/core/keras/quantizer.py @@ -37,7 +37,15 @@ def __init__( self.place = place self.granularity = granularity.value if isinstance(granularity, Enum) else granularity self.quantizer = create_quantizer( - self.k_init, self.i_init, self.f_init, self.overflow, self.round_mode, self.use_hgq, self.is_data, place + self.k_init, + self.i_init, + self.f_init, + self.overflow, + self.round_mode, + self.use_hgq, + self.is_data, + place, + granularity=self.granularity, ) self.is_pretraining = True self.hgq_gamma = hgq_gamma @@ -205,14 +213,41 @@ def get_config(self): return config -def create_hgq_parameters_quantizer(k, i, f, overflow, round_mode, place, gamma=1e-8): +def axis_kwargs_for_granularity(granularity, is_data): + """Translate a granularity into HGQ's (mutually exclusive) homogeneous/heterogeneous axis spec. + + HGQ only supports per_tensor and per_weight from the granularity enum: + - per_tensor: nothing varies -> heterogeneous_axis=() (one bitwidth for the whole tensor) + - per_weight: every element varies. For data we keep the batch axis (0) homogeneous via + homogeneous_axis=(0,); for weights nothing is shared via homogeneous_axis=(). + + per_channel is intentionally NOT supported for HGQ (the channel axis is layout-dependent, so we + don't guess it). + """ + if granularity == "per_tensor": + return {"heterogeneous_axis": ()} + if granularity == "per_weight": + return {"homogeneous_axis": (0,) if is_data else ()} + if granularity == "per_channel": + raise ValueError("per_channel granularity is not supported for HGQ. Use 'per_tensor' or 'per_weight'.") + raise ValueError(f"Unsupported granularity: {granularity}") + + +def create_hgq_parameters_quantizer(k, i, f, overflow, round_mode, place, axis_kwargs, gamma=1e-8): quantizer_config = QuantizerConfig( - q_type="kif", place=place, k0=k, i0=i, f0=f, overflow_mode=overflow, round_mode=round_mode, homogeneous_axis=() + q_type="kif", + place=place, + k0=k, + i0=i, + f0=f, + overflow_mode=overflow, + round_mode=round_mode, + **axis_kwargs, ) return HGQQuantizer(config=quantizer_config) -def create_hgq_data_quantizer(k, i, f, overflow, round_mode, gamma=1e-8): +def create_hgq_data_quantizer(k, i, f, overflow, round_mode, axis_kwargs, gamma=1e-8): quantizer_config = QuantizerConfig( q_type="kif", place="datalane", @@ -221,16 +256,19 @@ def create_hgq_data_quantizer(k, i, f, overflow, round_mode, gamma=1e-8): f0=f, overflow_mode=overflow, round_mode=round_mode, - homogeneous_axis=(0,), + **axis_kwargs, ) return HGQQuantizer(config=quantizer_config) -def create_quantizer(k, i, f, overflow, round_mode, is_heterogeneous, is_data, place="datalane", gamma=1e-8): +def create_quantizer( + k, i, f, overflow, round_mode, is_heterogeneous, is_data, place="datalane", granularity="per_weight", gamma=1e-8 +): if is_heterogeneous: + axis_kwargs = axis_kwargs_for_granularity(granularity, is_data) if is_data: - return create_hgq_data_quantizer(k, i, f, overflow, round_mode, gamma=gamma) + return create_hgq_data_quantizer(k, i, f, overflow, round_mode, axis_kwargs, gamma=gamma) else: - return create_hgq_parameters_quantizer(k, i, f, overflow, round_mode, place, gamma=gamma) + return create_hgq_parameters_quantizer(k, i, f, overflow, round_mode, place, axis_kwargs, gamma=gamma) else: return get_fixed_quantizer(round_mode=round_mode, overflow_mode=overflow) diff --git a/src/pquant/core/torch/activations.py b/src/pquant/core/torch/activations.py index 61347ea..4aa1773 100644 --- a/src/pquant/core/torch/activations.py +++ b/src/pquant/core/torch/activations.py @@ -107,6 +107,7 @@ def check_is_built(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.dynamic_data, + granularity=self.config.quantization_parameters.granularity, ) self.input_quantizer = Quantizer( k=self.k_input, @@ -119,6 +120,7 @@ def check_is_built(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.dynamic_data, + granularity=self.config.quantization_parameters.granularity, ) if self.use_hgq: self.input_quantizer.quantizer.build(input_shape) diff --git a/src/pquant/core/torch/hgq_quantizer.py b/src/pquant/core/torch/hgq_quantizer.py index c10ab55..26a468c 100644 --- a/src/pquant/core/torch/hgq_quantizer.py +++ b/src/pquant/core/torch/hgq_quantizer.py @@ -44,8 +44,12 @@ class HGQQuantizer(nn.Module): round_mode : str One of 'RND', 'RND_CONV', 'TRN', etc. is_data : bool - True β†’ data/activation quantizer (homogeneous over batch axis 0). - False β†’ weight/bias quantizer (fully heterogeneous, per-element). + True β†’ data/activation quantizer (batch axis 0 always homogeneous; + per_channel keys on channel axis 1). + False β†’ weight/bias quantizer (per_channel keys on output-channel axis 0). + granularity : str + One of 'per_tensor' (one shared bit-width) or 'per_weight' (one per element). + Controls which axes are shared. per_channel is not supported. gamma : float L1 regularisation coefficient on bit-widths. i_decay_speed : float @@ -73,6 +77,7 @@ def __init__( overflow_mode: str, round_mode: str, is_data: bool, + granularity: str = "per_weight", gamma: float = 1e-8, i_decay_speed: float = float("inf"), i_min: float = -23.0, @@ -92,6 +97,7 @@ def __init__( self.overflow_mode = overflow_mode.upper() self.round_mode = round_mode.upper() self.is_data = is_data + self.granularity = granularity self.gamma = gamma self.i_decay_speed = i_decay_speed self.i_min = i_min @@ -129,10 +135,9 @@ def build(self, input_shape: tuple) -> None: parameters, not the scalar placeholders from __init__. """ device = self._k.device + self.homogeneous_axis = self._homogeneous_axis(len(input_shape)) bw_shape = self._infer_bw_shape(input_shape) - self.homogeneous_axis = (0,) if self.is_data else () - # k: non-trainable sign-bit buffer self.register_buffer("_k", torch.full(bw_shape, self.k0, device=device)) @@ -146,15 +151,23 @@ def build(self, input_shape: tuple) -> None: self._built = True + def _homogeneous_axis(self, ndim: int) -> tuple[int, ...]: + """Axes shared (collapsed to 1 in the bit-width tensor), derived from `granularity`. + + HGQ supports only per_tensor and per_weight (data always shares the batch axis 0). + per_channel is intentionally unsupported (the channel axis is layout-dependent). + """ + if self.granularity == "per_tensor": + return tuple(range(ndim)) + if self.granularity == "per_weight": + return (0,) if self.is_data else () + if self.granularity == "per_channel": + raise ValueError("per_channel granularity is not supported for HGQ. Use 'per_tensor' or 'per_weight'.") + raise ValueError(f"Unsupported granularity: {self.granularity}") + def _infer_bw_shape(self, input_shape: tuple) -> tuple: - """Shape of bit-width parameter tensors given input tensor shape.""" - if self.is_data: - # Batch axis (0) is homogeneous β†’ dimension 0 collapses to 1. - shape = list(input_shape) - shape[0] = 1 - return tuple(shape) - # Fully heterogeneous (per-parameter): same shape as input. - return tuple(input_shape) + """Shape of bit-width parameter tensors: homogeneous axes collapse to 1.""" + return tuple(1 if ax in self.homogeneous_axis else d for ax, d in enumerate(input_shape)) # ------------------------------------------------------------------ # Properties diff --git a/src/pquant/core/torch/layers.py b/src/pquant/core/torch/layers.py index 64643fe..3357185 100644 --- a/src/pquant/core/torch/layers.py +++ b/src/pquant/core/torch/layers.py @@ -31,6 +31,10 @@ def __init__( weight_quant_bits: Tuple[T, T, T] = None, bias_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + weight_quant_granularity=None, + in_quant_granularity=None, + bias_quant_granularity=None, + out_quant_granularity=None, *args, **kwargs, ): @@ -78,6 +82,12 @@ def __init__( self.use_fitcompress = config.fitcompress_parameters.enable_fitcompress self.hgq_gamma = config.quantization_parameters.hgq_gamma self.granularity = config.quantization_parameters.granularity + self.weight_quant_granularity = ( + weight_quant_granularity if weight_quant_granularity is not None else self.granularity + ) + self.in_quant_granularity = in_quant_granularity if in_quant_granularity is not None else self.granularity + self.bias_quant_granularity = bias_quant_granularity if bias_quant_granularity is not None else self.granularity + self.out_quant_granularity = out_quant_granularity if out_quant_granularity is not None else self.granularity self.final_compression_done = False self.built = False self.parallelization_factor = -1 @@ -105,6 +115,7 @@ def check_is_built(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, + granularity=self.in_quant_granularity, ) self.weight_quantizer = Quantizer( k=torch.tensor(self.k_weight), @@ -115,7 +126,7 @@ def check_is_built(self, input_shape): is_heterogeneous=self.use_hgq, is_data=False, hgq_gamma=self.hgq_gamma, - granularity=self.granularity, + granularity=self.weight_quant_granularity, place="weight", ) @@ -128,6 +139,7 @@ def check_is_built(self, input_shape): is_heterogeneous=self.use_hgq, is_data=False, hgq_gamma=self.hgq_gamma, + granularity=self.bias_quant_granularity, place="bias", ) if self.quantize_output: @@ -142,6 +154,7 @@ def check_is_built(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, + granularity=self.out_quant_granularity, ) self.n_parallel = ops.prod(tuple(input_shape)[1:-1]) @@ -244,6 +257,10 @@ def __init__( weight_quant_bits: Tuple[T, T, T] = None, bias_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + weight_quant_granularity=None, + in_quant_granularity=None, + bias_quant_granularity=None, + out_quant_granularity=None, **kwargs, ): super().__init__( @@ -261,6 +278,10 @@ def __init__( weight_quant_bits=weight_quant_bits, bias_quant_bits=bias_quant_bits, out_quant_bits=out_quant_bits, + weight_quant_granularity=weight_quant_granularity, + in_quant_granularity=in_quant_granularity, + bias_quant_granularity=bias_quant_granularity, + out_quant_granularity=out_quant_granularity, **kwargs, ) self.in_features = in_features @@ -358,6 +379,10 @@ def __init__( weight_quant_bits: Tuple[T, T, T] = None, bias_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + weight_quant_granularity=None, + in_quant_granularity=None, + bias_quant_granularity=None, + out_quant_granularity=None, **kwargs, ): super().__init__( @@ -381,6 +406,10 @@ def __init__( weight_quant_bits=weight_quant_bits, bias_quant_bits=bias_quant_bits, out_quant_bits=out_quant_bits, + weight_quant_granularity=weight_quant_granularity, + in_quant_granularity=in_quant_granularity, + bias_quant_granularity=bias_quant_granularity, + out_quant_granularity=out_quant_granularity, **kwargs, ) self.use_fitcompress = config.fitcompress_parameters.enable_fitcompress @@ -499,6 +528,10 @@ def __init__( weight_quant_bits: Tuple[T, T, T] = None, bias_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + weight_quant_granularity=None, + in_quant_granularity=None, + bias_quant_granularity=None, + out_quant_granularity=None, **kwargs, ): super().__init__( @@ -522,6 +555,10 @@ def __init__( weight_quant_bits=weight_quant_bits, bias_quant_bits=bias_quant_bits, out_quant_bits=out_quant_bits, + weight_quant_granularity=weight_quant_granularity, + in_quant_granularity=in_quant_granularity, + bias_quant_granularity=bias_quant_granularity, + out_quant_granularity=out_quant_granularity, **kwargs, ) self.use_fitcompress = config.fitcompress_parameters.enable_fitcompress @@ -625,6 +662,8 @@ def __init__( quantize_output=False, in_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + in_quant_granularity=None, + out_quant_granularity=None, **kwargs, ): super().__init__(**kwargs) @@ -654,6 +693,10 @@ def __init__( self.saved_inputs = [] self.quantize_input = quantize_input self.quantize_output = quantize_output + # Optional per-quantizer granularity override; None β†’ inherit config granularity. + granularity = config.quantization_parameters.granularity + self.in_quant_granularity = in_quant_granularity if in_quant_granularity is not None else granularity + self.out_quant_granularity = out_quant_granularity if out_quant_granularity is not None else granularity def build(self, input_shape): self.input_quantizer = Quantizer( @@ -667,6 +710,7 @@ def build(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, + granularity=self.in_quant_granularity, ) self.output_quantizer = Quantizer( k=torch.tensor(self.k_output), @@ -679,6 +723,7 @@ def build(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, + granularity=self.out_quant_granularity, ) self.input_shape = (1,) + input_shape[1:] @@ -743,6 +788,8 @@ def __init__( quantize_output=False, in_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + in_quant_granularity=None, + out_quant_granularity=None, **kwargs, ): super().__init__( @@ -756,6 +803,8 @@ def __init__( quantize_output=quantize_output, in_quant_bits=in_quant_bits, out_quant_bits=out_quant_bits, + in_quant_granularity=in_quant_granularity, + out_quant_granularity=out_quant_granularity, **kwargs, ) @@ -780,6 +829,8 @@ def __init__( quantize_output=False, in_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, + in_quant_granularity=None, + out_quant_granularity=None, **kwargs, ): super().__init__( @@ -794,6 +845,8 @@ def __init__( quantize_output=quantize_output, in_quant_bits=in_quant_bits, out_quant_bits=out_quant_bits, + in_quant_granularity=in_quant_granularity, + out_quant_granularity=out_quant_granularity, **kwargs, ) @@ -819,6 +872,9 @@ def __init__( in_quant_bits: Tuple[T, T, T] = None, weight_quant_bits: Tuple[T, T, T] = None, bias_quant_bits: Tuple[T, T, T] = None, + in_quant_granularity=None, + weight_quant_granularity=None, + bias_quant_granularity=None, ): super().__init__(num_features, eps, momentum, affine, track_running_stats, device=device, dtype=dtype) if in_quant_bits is not None: @@ -850,6 +906,10 @@ def __init__( self.use_fitcompress = config.fitcompress_parameters.enable_fitcompress self.config = config self.quantize_input = quantize_input + granularity = config.quantization_parameters.granularity + self.in_quant_granularity = in_quant_granularity if in_quant_granularity is not None else granularity + self.weight_quant_granularity = weight_quant_granularity if weight_quant_granularity is not None else granularity + self.bias_quant_granularity = bias_quant_granularity if bias_quant_granularity is not None else granularity self._weight = nn.Parameter(self.weight.clone()).to(self.weight.device) self.register_parameter("_weight", self._weight) if self.bias is not None: @@ -878,6 +938,7 @@ def check_is_built(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, + granularity=self.in_quant_granularity, ) self.weight_quantizer = Quantizer( k=torch.tensor(self.k_weight), @@ -888,6 +949,7 @@ def check_is_built(self, input_shape): is_data=False, is_heterogeneous=self.use_hgq, place="weight", + granularity=self.weight_quant_granularity, ) self.bias_quantizer = Quantizer( k=torch.tensor(self.k_bias), @@ -898,6 +960,7 @@ def check_is_built(self, input_shape): is_data=False, is_heterogeneous=self.use_hgq, place="bias", + granularity=self.bias_quant_granularity, ) if self.use_hgq: self.input_quantizer.quantizer.build(input_shape) @@ -982,6 +1045,9 @@ def __init__( in_quant_bits: Tuple[T, T, T] = None, weight_quant_bits: Tuple[T, T, T] = None, bias_quant_bits: Tuple[T, T, T] = None, + in_quant_granularity=None, + weight_quant_granularity=None, + bias_quant_granularity=None, ): super().__init__(num_features, eps, momentum, affine, track_running_stats, device=device, dtype=dtype) if in_quant_bits is not None: @@ -1013,6 +1079,10 @@ def __init__( self.use_fitcompress = config.fitcompress_parameters.enable_fitcompress self.config = config self.quantize_input = quantize_input + granularity = config.quantization_parameters.granularity + self.in_quant_granularity = in_quant_granularity if in_quant_granularity is not None else granularity + self.weight_quant_granularity = weight_quant_granularity if weight_quant_granularity is not None else granularity + self.bias_quant_granularity = bias_quant_granularity if bias_quant_granularity is not None else granularity self._weight = nn.Parameter(self.weight.clone()).to(self.weight.device) self.register_parameter("_weight", self._weight) if self.bias is not None: @@ -1042,6 +1112,7 @@ def check_is_built(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, + granularity=self.in_quant_granularity, ) self.weight_quantizer = Quantizer( k=torch.tensor(self.k_weight), @@ -1052,6 +1123,7 @@ def check_is_built(self, input_shape): is_data=False, is_heterogeneous=self.use_hgq, place="weight", + granularity=self.weight_quant_granularity, ) self.bias_quantizer = Quantizer( k=torch.tensor(self.k_bias), @@ -1062,6 +1134,7 @@ def check_is_built(self, input_shape): is_data=False, is_heterogeneous=self.use_hgq, place="bias", + granularity=self.bias_quant_granularity, ) if self.use_hgq: self.input_quantizer.quantizer.build(input_shape) @@ -1147,6 +1220,10 @@ def __init__( out_quant_bits: Tuple[T, T, T] = None, weight_quant_bits: Tuple[T, T, T] = None, bias_quant_bits: Tuple[T, T, T] = None, + in_quant_granularity=None, + out_quant_granularity=None, + weight_quant_granularity=None, + bias_quant_granularity=None, ): try: super().__init__(normalized_shape, eps, elementwise_affine, bias, device=device, dtype=dtype) @@ -1190,6 +1267,11 @@ def __init__( self.config = config self.quantize_input = quantize_input self.quantize_output = quantize_output + granularity = config.quantization_parameters.granularity + self.in_quant_granularity = in_quant_granularity if in_quant_granularity is not None else granularity + self.out_quant_granularity = out_quant_granularity if out_quant_granularity is not None else granularity + self.weight_quant_granularity = weight_quant_granularity if weight_quant_granularity is not None else granularity + self.bias_quant_granularity = bias_quant_granularity if bias_quant_granularity is not None else granularity if self.weight is not None: self._weight = nn.Parameter(self.weight.clone()).to(self.weight.device) self.register_parameter("_weight", self._weight) @@ -1221,6 +1303,7 @@ def check_is_built(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, + granularity=self.in_quant_granularity, ) self.output_quantizer = Quantizer( k=torch.tensor(self.k_output), @@ -1233,6 +1316,7 @@ def check_is_built(self, input_shape): hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, + granularity=self.out_quant_granularity, ) self.weight_quantizer = Quantizer( k=torch.tensor(self.k_weight), @@ -1243,6 +1327,7 @@ def check_is_built(self, input_shape): is_data=False, is_heterogeneous=self.use_hgq, place="weight", + granularity=self.weight_quant_granularity, ) self.bias_quantizer = Quantizer( k=torch.tensor(self.k_bias), @@ -1253,6 +1338,7 @@ def check_is_built(self, input_shape): is_data=False, is_heterogeneous=self.use_hgq, place="bias", + granularity=self.bias_quant_granularity, ) if self.use_hgq: self.input_quantizer.quantizer.build(input_shape) @@ -1385,6 +1471,9 @@ def __init__( bias_quant_bits: Tuple[T, T, T] = None, out_quant_bits: Tuple[T, T, T] = None, attn_quant_bits: Tuple[T, T, T] = None, + in_quant_granularity=None, + out_quant_granularity=None, + param_quant_granularity=None, **kwargs, ): super().__init__(**kwargs) @@ -1402,19 +1491,33 @@ def __init__( kdim = kdim if kdim is not None else embed_dim vdim = vdim if vdim is not None else embed_dim + self.in_quant_granularity = in_quant_granularity + self.out_quant_granularity = out_quant_granularity + self.param_quant_granularity = param_quant_granularity proj_kwargs = dict( bias=bias, in_quant_bits=in_quant_bits, weight_quant_bits=weight_quant_bits, bias_quant_bits=bias_quant_bits, out_quant_bits=out_quant_bits, + weight_quant_granularity=param_quant_granularity, + bias_quant_granularity=param_quant_granularity, + ) + + qkv_kwargs = dict( + quantize_input=quantize_input, quantize_output=True, in_quant_granularity=in_quant_granularity, **proj_kwargs ) - qkv_kwargs = dict(quantize_input=quantize_input, quantize_output=True, **proj_kwargs) self.q_proj = PQDense(config, embed_dim, embed_dim, enable_pruning=False, **qkv_kwargs) self.k_proj = PQDense(config, kdim, embed_dim, enable_pruning=False, **qkv_kwargs) self.v_proj = PQDense(config, vdim, embed_dim, enable_pruning=False, **qkv_kwargs) self.out_proj = PQDense( - config, embed_dim, embed_dim, quantize_input=True, quantize_output=quantize_output, **proj_kwargs + config, + embed_dim, + embed_dim, + quantize_input=True, + quantize_output=quantize_output, + out_quant_granularity=out_quant_granularity, + **proj_kwargs, ) self.attn_dropout = None diff --git a/src/pquant/core/torch/quantizer.py b/src/pquant/core/torch/quantizer.py index e9bdb1d..7f0a0e9 100644 --- a/src/pquant/core/torch/quantizer.py +++ b/src/pquant/core/torch/quantizer.py @@ -37,7 +37,15 @@ def __init__( self.b = torch.nn.Parameter(torch.tensor(i + k + f), requires_grad=False) self.granularity = granularity.value if isinstance(granularity, Enum) else granularity self.quantizer = create_quantizer( - self.k, i, f, self.overflow, self.round_mode, self.use_hgq, self.is_data, hgq_gamma + self.k, + i, + f, + self.overflow, + self.round_mode, + self.use_hgq, + self.is_data, + granularity=self.granularity, + gamma=hgq_gamma, ) self.is_pretraining = True self.hgq_gamma = hgq_gamma @@ -156,11 +164,17 @@ def reload_from_local(self): self.quantizer.set_bits(self.i, self.f) -def create_quantizer(k, i, f, overflow, round_mode, is_heterogeneous, is_data, gamma=1e-8): +def create_quantizer(k, i, f, overflow, round_mode, is_heterogeneous, is_data, granularity="per_weight", gamma=1e-8): if is_heterogeneous: - if is_data: - return HGQQuantizer(k0=k, i0=i, f0=f, overflow_mode=overflow, round_mode=round_mode, is_data=True, gamma=gamma) - else: - return HGQQuantizer(k0=k, i0=i, f0=f, overflow_mode=overflow, round_mode=round_mode, is_data=False, gamma=gamma) + return HGQQuantizer( + k0=k, + i0=i, + f0=f, + overflow_mode=overflow, + round_mode=round_mode, + is_data=is_data, + granularity=granularity, + gamma=gamma, + ) else: return get_fixed_quantizer(round_mode=round_mode, overflow_mode=overflow) diff --git a/tests/conftest.py b/tests/conftest.py index 479ab00..f581100 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,7 +41,12 @@ def configure_backend(): match backend: case 'tensorflow': - pass + import tensorflow as tf + + # Use full float32 matmul precision to match numpy/onnxruntime. Without this, TF uses + # TF32 on Ampere+ GPUs and the ~1e-3 relative error exceeds tight test tolerances + # (e.g. the kerasβ†’ONNX parity tests). Mirrors the torch 'highest' setting below. + tf.config.experimental.enable_tensor_float_32_execution(False) case 'torch': import torch diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 3121670..66119ec 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -9,5 +9,9 @@ KERAS_BACKEND="torch" pytest test_wanda.py pytest test_keras_compression_layers.py DATA_FORMAT=channels_last pytest test_keras_compression_layers.py KERAS_BACKEND="torch" pytest test_torch_compression_layers.py -pytest test_torch_onnx_converter.py +KERAS_BACKEND=torch pytest test_torch_onnx_converter.py pytest test_keras_onnx_converter.py +KERAS_BACKEND=torch pytest test_torch_alkaid_conversion.py +pytest test_keras_alkaid_conversion.py +KERAS_BACKEND=torch pytest test_hgq_torch.py +pytest test_hgq_keras.py diff --git a/tests/test_hgq_keras.py b/tests/test_hgq_keras.py new file mode 100644 index 0000000..a4dcf51 --- /dev/null +++ b/tests/test_hgq_keras.py @@ -0,0 +1,241 @@ +"""Tests for HGQ (high granularity quantization) per-tensor / per-channel / per-weight granularity. + +The granularity controls the shape of the trainable bitwidth tensors (`i` and `f`) inside the +HGQ quantizer: + - per_tensor: a single shared value for the whole tensor -> shape collapses to all-ones + - per_channel: one value per output channel -> only the output-channel axis is kept + - per_weight: one value per element -> shape matches the tensor itself + (for data the batch axis is always shared, so it collapses to 1) + +In Keras, kernels are stored output-channel-last and we assume channels_last data, so the +output-channel axis is -1 for both weights and activations. +""" + +import keras +import numpy as np +import pytest +from keras import ops + +from pquant import pdp_config +from pquant.core.keras.quantizer import Quantizer +from pquant.layers import ( + PQAvgPool1d, + PQBatchNormalization, + PQConv1d, + PQConv2d, + PQDense, + PQMultiheadAttention, +) + +BATCH_SIZE = 4 +IN_FEATURES = 16 +OUT_FEATURES = 32 +KERNEL_SIZE = 3 +STEPS = 8 + +# HGQ supports only per_tensor and per_weight; per_channel is rejected. +GRANULARITIES = ["per_tensor", "per_weight"] + + +@pytest.fixture(autouse=True) +def run_around_tests(): + keras.backend.clear_session() + + +def hgq_config(granularity): + """A PQConfig with high granularity quantization enabled for the given granularity.""" + config = pdp_config() + config.quantization_parameters.use_high_granularity_quantization = True + config.quantization_parameters.enable_quantization = True + config.quantization_parameters.granularity = granularity + return config + + +def shape_of(tensor): + return tuple(int(d) for d in ops.shape(tensor)) + + +def is_single_value(shape): + """per_tensor: every axis collapsed to 1.""" + return int(np.prod(shape)) == 1 + + +# ----------------------------------------------------------------------------- weights + + +def assert_weight_granularity(layer, granularity, kernel_shape): + """The weight quantizer's i/f bitwidth tensors must match the expected shape for `granularity`.""" + _, i, f = layer.get_weight_quantization_bits() + for name, t in (("i", i), ("f", f)): + shape = shape_of(t) + if granularity == "per_tensor": + assert is_single_value(shape), f"weight {name}: expected single value, got {shape}" + else: # per_weight + assert shape == kernel_shape, f"weight {name}: expected {kernel_shape}, got {shape}" + + +@pytest.mark.parametrize("granularity", GRANULARITIES) +def test_dense_weight_granularity(granularity): + layer = PQDense(hgq_config(granularity), units=OUT_FEATURES) + layer.build((BATCH_SIZE, IN_FEATURES)) + assert_weight_granularity(layer, granularity, kernel_shape=(IN_FEATURES, OUT_FEATURES)) + + +@pytest.mark.parametrize("granularity", GRANULARITIES) +def test_conv1d_weight_granularity(granularity): + layer = PQConv1d(hgq_config(granularity), filters=OUT_FEATURES, kernel_size=KERNEL_SIZE, data_format="channels_last") + layer.build((BATCH_SIZE, STEPS, IN_FEATURES)) + assert_weight_granularity(layer, granularity, kernel_shape=(KERNEL_SIZE, IN_FEATURES, OUT_FEATURES)) + + +@pytest.mark.parametrize("granularity", GRANULARITIES) +def test_conv2d_weight_granularity(granularity): + layer = PQConv2d(hgq_config(granularity), filters=OUT_FEATURES, kernel_size=KERNEL_SIZE, data_format="channels_last") + layer.build((BATCH_SIZE, STEPS, STEPS, IN_FEATURES)) + assert_weight_granularity(layer, granularity, kernel_shape=(KERNEL_SIZE, KERNEL_SIZE, IN_FEATURES, OUT_FEATURES)) + + +def test_per_channel_rejected_for_hgq(): + # per_channel is not a valid HGQ granularity; constructing the HGQ weight quantizer must raise. + with pytest.raises(ValueError, match="per_channel"): + PQDense(hgq_config("per_channel"), units=OUT_FEATURES) + + +# ------------------------------------------------------------------------------- data +# A data quantizer built on a tensor shaped like a layer's output. The batch axis is always +# shared, so per_weight keeps every non-batch axis and collapses only the batch axis to 1. + + +def assert_data_granularity(output_shape, granularity): + quantizer = Quantizer(k=0.0, i=0.0, f=7.0, is_heterogeneous=True, is_data=True, granularity=granularity) + quantizer.build(output_shape) + _, i, f = quantizer.get_quantization_bits() + for name, t in (("i", i), ("f", f)): + shape = shape_of(t) + if granularity == "per_tensor": + assert is_single_value(shape), f"data {name}: expected single value, got {shape}" + else: # per_weight: batch axis shared, rest per-element + assert shape == (1,) + tuple(output_shape[1:]), f"data {name}: expected batch-collapsed, got {shape}" + + +@pytest.mark.parametrize("granularity", GRANULARITIES) +def test_dense_data_granularity(granularity): + assert_data_granularity((BATCH_SIZE, OUT_FEATURES), granularity) + + +@pytest.mark.parametrize("granularity", GRANULARITIES) +def test_conv1d_data_granularity(granularity): + # channels_last, "valid" padding output length + assert_data_granularity((BATCH_SIZE, STEPS - KERNEL_SIZE + 1, OUT_FEATURES), granularity) + + +@pytest.mark.parametrize("granularity", GRANULARITIES) +def test_conv2d_data_granularity(granularity): + out_len = STEPS - KERNEL_SIZE + 1 + assert_data_granularity((BATCH_SIZE, out_len, out_len, OUT_FEATURES), granularity) + + +# --------------------------------------------------- per-quantizer granularity override +# Each quantizer follows the config granularity unless its per-quantizer override is set. + + +def test_per_quantizer_granularity_override(): + # input per_tensor, everything else per_weight. + config = hgq_config("per_weight") + layer = PQDense(config, units=OUT_FEATURES, quantize_output=True, in_quant_granularity="per_tensor") + layer.build((BATCH_SIZE, IN_FEATURES)) + + # weight follows config (per_weight) -> full kernel shape + _, wi, _ = layer.get_weight_quantization_bits() + assert shape_of(wi) == (IN_FEATURES, OUT_FEATURES) + # input overridden to per_tensor -> single value + _, ii, _ = layer.get_input_quantization_bits() + assert is_single_value(shape_of(ii)) + # output keeps config per_weight -> batch-collapsed data shape + _, oi, _ = layer.get_output_quantization_bits() + assert shape_of(oi) == (1, OUT_FEATURES) + + +def test_per_quantizer_granularity_defaults_to_config(): + config = hgq_config("per_tensor") + layer = PQDense(config, units=OUT_FEATURES, weight_quant_granularity="per_weight") + layer.build((BATCH_SIZE, IN_FEATURES)) + # weight overridden to per_weight; input uses config per_tensor + _, wi, _ = layer.get_weight_quantization_bits() + assert shape_of(wi) == (IN_FEATURES, OUT_FEATURES) + _, ii, _ = layer.get_input_quantization_bits() + assert is_single_value(shape_of(ii)) + + +# --------------------------------------------------- granularity override on boundary layers +# A model can start/end with a BatchNorm / AvgPool whose input/output quantizer is effectively +# the model's I/O quantizer, so that must be overridable independently of the config granularity. + + +def test_batchnorm_input_granularity_override(): + config = hgq_config("per_weight") + layer = PQBatchNormalization(config, in_quant_granularity="per_tensor") + layer.build((BATCH_SIZE, IN_FEATURES)) + # input overridden to per_tensor -> single value + _, ii, _ = layer.get_input_quantization_bits() + assert is_single_value(shape_of(ii)) + # weight follows config per_weight -> one value per feature + _, wi, _ = layer.get_weight_quantization_bits() + assert shape_of(wi) == (IN_FEATURES,) + + +def test_avgpool_io_granularity_override(): + config = hgq_config("per_weight") + layer = PQAvgPool1d( + config, + pool_size=2, + quantize_output=True, + in_quant_granularity="per_tensor", + out_quant_granularity="per_tensor", + ) + layer.build((BATCH_SIZE, STEPS, OUT_FEATURES)) + _, ii, _ = layer.get_input_quantization_bits() + assert is_single_value(shape_of(ii)) # input overridden to per_tensor + _, oi, _ = layer.get_output_quantization_bits() + assert is_single_value(shape_of(oi)) # output overridden to per_tensor + + +def test_mha_io_param_granularity_override(): + config = hgq_config("per_weight") + # in/out are the model-boundary granularities; param (weight+bias) is uniform across projections. + layer = PQMultiheadAttention( + config, + embed_dim=IN_FEATURES, + num_heads=4, + quantize_output=True, + in_quant_granularity="per_tensor", + out_quant_granularity="per_tensor", + ) + # Build the projections directly (a full forward would hit an unrelated dtype issue in + # PQDense.ebops on 3-D input); granularity shapes are fixed at build time. + for proj in (layer.q_proj, layer.k_proj, layer.v_proj, layer.out_proj): + proj.build((2, 5, IN_FEATURES)) + # Q/K/V projection inputs (boundary) overridden to per_tensor + for proj in (layer.q_proj, layer.k_proj, layer.v_proj): + _, ii, _ = proj.get_input_quantization_bits() + assert is_single_value(shape_of(ii)) + # out_proj output (boundary) overridden to per_tensor + _, oi, _ = layer.out_proj.get_output_quantization_bits() + assert is_single_value(shape_of(oi)) + # weights follow param granularity = config per_weight + _, wi, _ = layer.q_proj.get_weight_quantization_bits() + assert shape_of(wi) == (IN_FEATURES, IN_FEATURES) + # out_proj input is internal -> stays config per_weight (batch-collapsed, not single) + _, oii, _ = layer.out_proj.get_input_quantization_bits() + assert not is_single_value(shape_of(oii)) + + +def test_mha_qkv_always_output_quantized(): + config = hgq_config("per_tensor") + # MHA-level quantize_output=False (default): Q/K/V outputs are matmul operands and stay + # output-quantized; only out_proj follows the MHA-level flag. + layer = PQMultiheadAttention(config, embed_dim=IN_FEATURES, num_heads=4) + assert layer.q_proj.quantize_output is True + assert layer.k_proj.quantize_output is True + assert layer.v_proj.quantize_output is True + assert layer.out_proj.quantize_output is False diff --git a/tests/test_hgq_torch.py b/tests/test_hgq_torch.py index 9166725..c039ced 100644 --- a/tests/test_hgq_torch.py +++ b/tests/test_hgq_torch.py @@ -6,10 +6,6 @@ training trajectory when fed the same data with the same initial state. """ -import os - -os.environ.setdefault("KERAS_BACKEND", "torch") - import pytest # noqa: E402 import torch # noqa: E402 @@ -22,6 +18,9 @@ RTOL = 1e-4 ATOL = 1e-5 +# HGQ supports only per_tensor and per_weight; per_channel is rejected. +GRANULARITIES = ["per_tensor", "per_weight"] + # --------------------------------------------------------------------------- # Helpers @@ -215,6 +214,247 @@ def test_training_trajectory_matches(): assert diff_out < 0.1, f"trained outputs diverged: max diff = {diff_out:.4f}" +# --------------------------------------------------------------------------- +# Granularity β†’ bit-width tensor shape +# +# per_tensor : one shared bit-width -> shape collapses to all-ones +# per_weight : one per element -> shape matches the tensor +# (data always shares the batch axis, so axis 0 collapses to 1) +# per_channel is not supported via granularity (see axis-override tests). +# --------------------------------------------------------------------------- + + +def _is_single_value(shape): + return int(torch.tensor(shape).prod().item()) == 1 + + +def _build_hgq(shape, is_data, granularity): + q = HGQQuantizer(k0=1, i0=2, f0=4, overflow_mode="SAT", round_mode="RND", is_data=is_data, granularity=granularity) + q.build(shape) + return tuple(q.i.shape), tuple(q.f.shape) + + +# weight layout (output-channel first): Linear (out, in), Conv1d (out, in, k), Conv2d (out, in, kH, kW) +WEIGHT_SHAPES = [(32, 16), (32, 16, 3), (32, 16, 3, 3)] +# data layout (channels-first): (batch, channels, *spatial) +DATA_SHAPES = [(4, 32), (4, 32, 6), (4, 32, 6, 6)] + + +@pytest.mark.parametrize("granularity", GRANULARITIES) +@pytest.mark.parametrize("shape", WEIGHT_SHAPES) +def test_weight_granularity_shape(granularity, shape): + for bw_shape in _build_hgq(shape, is_data=False, granularity=granularity): + if granularity == "per_tensor": + assert _is_single_value(bw_shape), f"expected single value, got {bw_shape}" + else: # per_weight + assert bw_shape == shape, f"expected {shape}, got {bw_shape}" + + +@pytest.mark.parametrize("granularity", GRANULARITIES) +@pytest.mark.parametrize("shape", DATA_SHAPES) +def test_data_granularity_shape(granularity, shape): + for bw_shape in _build_hgq(shape, is_data=True, granularity=granularity): + if granularity == "per_tensor": + assert _is_single_value(bw_shape), f"expected single value, got {bw_shape}" + else: # per_weight: batch axis shared, rest per-element + assert bw_shape == (1,) + shape[1:], f"expected batch-collapsed, got {bw_shape}" + + +@pytest.mark.parametrize("is_data", [False, True]) +def test_per_channel_rejected_for_hgq(is_data): + # per_channel is not a valid HGQ granularity; building must raise. + q = HGQQuantizer(k0=1, i0=2, f0=4, overflow_mode="SAT", round_mode="RND", is_data=is_data, granularity="per_channel") + with pytest.raises(ValueError, match="per_channel"): + q.build((32, 16, 3, 3)) + + +# --------------------------------------------------------------------------- +# Per-quantizer granularity override +# --------------------------------------------------------------------------- + + +def test_per_quantizer_granularity_override(): + from pquant.core.torch.layers import PQDense + + # input per_tensor, everything else per_weight + layer = PQDense( + _hgq_config("per_weight"), + in_features=16, + out_features=32, + quantize_output=True, + in_quant_granularity="per_tensor", + ) + layer(torch.randn(4, 16)) + _, wi, _ = layer.get_weight_quantization_bits() + assert tuple(wi.shape) == (32, 16) # weight follows config per_weight + _, ii, _ = layer.get_input_quantization_bits() + assert _is_single_value(tuple(ii.shape)) # input overridden to per_tensor + _, oi, _ = layer.get_output_quantization_bits() + assert tuple(oi.shape) == (1, 32) # output keeps config per_weight (batch-collapsed) + + +def test_per_quantizer_granularity_defaults_to_config(): + from pquant.core.torch.layers import PQDense + + layer = PQDense(_hgq_config("per_tensor"), in_features=16, out_features=32, weight_quant_granularity="per_weight") + layer(torch.randn(4, 16)) + _, wi, _ = layer.get_weight_quantization_bits() + assert tuple(wi.shape) == (32, 16) # weight overridden to per_weight + _, ii, _ = layer.get_input_quantization_bits() + assert _is_single_value(tuple(ii.shape)) # input uses config per_tensor + + +# --------------------------------------------------------------------------- +# Per-quantizer granularity override on auxiliary (boundary) layers. +# A model can start/end with a BatchNorm / LayerNorm / AvgPool whose input/output +# quantizer is effectively the model's I/O quantizer, so that must be overridable. +# --------------------------------------------------------------------------- + + +def test_batchnorm_input_granularity_override(): + from pquant.core.torch.layers import PQBatchNorm1d + + # config per_weight, but this BN sits on the model boundary -> input per_tensor + layer = PQBatchNorm1d(_hgq_config("per_weight"), num_features=16, in_quant_granularity="per_tensor") + layer(torch.randn(4, 16)) + _, ii, _ = layer.get_input_quantization_bits() + assert _is_single_value(tuple(ii.shape)) # input overridden to per_tensor + _, wi, _ = layer.get_weight_quantization_bits() + assert tuple(wi.shape) == (16,) # weight follows config per_weight + + +def test_layernorm_io_granularity_override(): + from pquant.core.torch.layers import PQLayerNorm + + layer = PQLayerNorm( + _hgq_config("per_weight"), + normalized_shape=16, + quantize_output=True, + in_quant_granularity="per_tensor", + out_quant_granularity="per_tensor", + ) + layer(torch.randn(4, 16)) + _, ii, _ = layer.get_input_quantization_bits() + assert _is_single_value(tuple(ii.shape)) # input overridden to per_tensor + _, oi, _ = layer.get_output_quantization_bits() + assert _is_single_value(tuple(oi.shape)) # output overridden to per_tensor + _, wi, _ = layer.get_weight_quantization_bits() + assert tuple(wi.shape) == (16,) # weight follows config per_weight + + +def test_avgpool_io_granularity_override(): + from pquant.core.torch.layers import PQAvgPool1d + + layer = PQAvgPool1d( + _hgq_config("per_weight"), + kernel_size=2, + quantize_output=True, + in_quant_granularity="per_tensor", + out_quant_granularity="per_tensor", + ) + layer(torch.randn(4, 16, 8)) + _, ii, _ = layer.get_input_quantization_bits() + assert _is_single_value(tuple(ii.shape)) # input overridden to per_tensor + _, oi, _ = layer.get_output_quantization_bits() + assert _is_single_value(tuple(oi.shape)) # output overridden to per_tensor + + +def test_mha_io_param_granularity_override(): + from pquant.core.torch.layers import PQMultiheadAttention + + # in/out are the model-boundary granularities; param (weight+bias) is uniform across projections. + layer = PQMultiheadAttention( + _hgq_config("per_weight"), + embed_dim=16, + num_heads=4, + batch_first=True, + quantize_output=True, + in_quant_granularity="per_tensor", + out_quant_granularity="per_tensor", + ) + x = torch.randn(2, 5, 16) + layer(x, x, x) + # Q/K/V projection inputs (boundary) overridden to per_tensor + for proj in (layer.q_proj, layer.k_proj, layer.v_proj): + _, ii, _ = proj.get_input_quantization_bits() + assert _is_single_value(tuple(ii.shape)) + # out_proj output (boundary) overridden to per_tensor + _, oi, _ = layer.out_proj.get_output_quantization_bits() + assert _is_single_value(tuple(oi.shape)) + # weights follow param granularity = config per_weight + _, wi, _ = layer.q_proj.get_weight_quantization_bits() + assert tuple(wi.shape) == (16, 16) + # out_proj input is internal -> stays config per_weight (batch-collapsed, not single) + _, oii, _ = layer.out_proj.get_input_quantization_bits() + assert not _is_single_value(tuple(oii.shape)) + + +def test_mha_qkv_always_output_quantized(): + from pquant.core.torch.layers import PQMultiheadAttention + + # MHA-level quantize_output=False (default): Q/K/V outputs are matmul operands and stay + # output-quantized; only out_proj follows the MHA-level flag. + layer = PQMultiheadAttention(_hgq_config("per_tensor"), embed_dim=16, num_heads=4, batch_first=True) + assert layer.q_proj.quantize_output is True + assert layer.k_proj.quantize_output is True + assert layer.v_proj.quantize_output is True + assert layer.out_proj.quantize_output is False + + +def test_mha_param_granularity_override(): + from pquant.core.torch.layers import PQMultiheadAttention + + # param overrides weight+bias across all projections; inputs/outputs follow config per_tensor. + layer = PQMultiheadAttention( + _hgq_config("per_tensor"), + embed_dim=16, + num_heads=4, + batch_first=True, + param_quant_granularity="per_weight", + ) + x = torch.randn(2, 5, 16) + layer(x, x, x) + _, wi, _ = layer.out_proj.get_weight_quantization_bits() + assert tuple(wi.shape) == (16, 16) # weight overridden to per_weight + _, ii, _ = layer.q_proj.get_input_quantization_bits() + assert _is_single_value(tuple(ii.shape)) # input uses config per_tensor + + +# --------------------------------------------------------------------------- +# Granularity end-to-end through the PQ layers +# --------------------------------------------------------------------------- + + +def _hgq_config(granularity): + from pquant import pdp_config + + config = pdp_config() + config.quantization_parameters.use_high_granularity_quantization = True + config.quantization_parameters.enable_quantization = True + config.quantization_parameters.granularity = granularity + config.pruning_parameters.enable_pruning = False + return config + + +@pytest.mark.parametrize("granularity", GRANULARITIES) +def test_layer_weight_granularity(granularity): + from pquant.core.torch.layers import PQConv1d, PQConv2d, PQDense + + cases = [ + (PQDense(_hgq_config(granularity), in_features=16, out_features=32), torch.randn(4, 16), (32, 16)), + (PQConv1d(_hgq_config(granularity), 16, 32, kernel_size=3), torch.randn(4, 16, 8), (32, 16, 3)), + (PQConv2d(_hgq_config(granularity), 16, 32, kernel_size=3), torch.randn(4, 16, 8, 8), (32, 16, 3, 3)), + ] + for layer, x, weight_shape in cases: + layer(x) # triggers lazy build of the weight quantizer + _, i, f = layer.get_weight_quantization_bits() + for bw_shape in (tuple(i.shape), tuple(f.shape)): + if granularity == "per_tensor": + assert _is_single_value(bw_shape), f"{type(layer).__name__}: expected single value, got {bw_shape}" + else: # per_weight + assert bw_shape == weight_shape, f"{type(layer).__name__}: expected {weight_shape}, got {bw_shape}" + + # --------------------------------------------------------------------------- # Utility: locate f / i parameters inside the Keras hgq2 quantizer # --------------------------------------------------------------------------- diff --git a/tests/test_keras_compression_layers.py b/tests/test_keras_compression_layers.py index 2cee1b0..53d272b 100644 --- a/tests/test_keras_compression_layers.py +++ b/tests/test_keras_compression_layers.py @@ -1387,6 +1387,8 @@ def test_trigger_post_pretraining(config_pdp, conv2d_input): def test_hgq_weight_shape(config_pdp, dense_input): config_pdp.quantization_parameters.enable_quantization = True config_pdp.quantization_parameters.use_high_granularity_quantization = True + # Per-weight granularity β†’ one bit-width per kernel element. + config_pdp.quantization_parameters.granularity = "per_weight" inputs = keras.Input(shape=dense_input.shape[1:]) out = Dense(OUT_FEATURES, use_bias=False)(inputs) act1 = Activation("tanh")(out) diff --git a/tests/test_torch_compression_layers.py b/tests/test_torch_compression_layers.py index 668c968..e39ed66 100644 --- a/tests/test_torch_compression_layers.py +++ b/tests/test_torch_compression_layers.py @@ -604,6 +604,8 @@ def _to_bool(val): def test_hgq_weight_shape(config_pdp, dense_input): config_pdp.quantization_parameters.enable_quantization = True config_pdp.quantization_parameters.use_high_granularity_quantization = True + # Per-weight granularity β†’ one bit-width per weight element. + config_pdp.quantization_parameters.granularity = "per_weight" layer = Linear(IN_FEATURES, OUT_FEATURES, bias=False) layer2 = Linear(OUT_FEATURES, OUT_FEATURES, bias=False) model = TestModel2(layer, layer2, "relu", "tanh") @@ -618,6 +620,8 @@ def test_hgq_weight_shape(config_pdp, dense_input): def test_qbn_build(config_pdp, conv2d_input): config_pdp.quantization_parameters.enable_quantization = True config_pdp.quantization_parameters.use_high_granularity_quantization = True + # Per-weight granularity β†’ one bit-width per conv-kernel element. + config_pdp.quantization_parameters.granularity = "per_weight" layer = Conv2d(IN_FEATURES, OUT_FEATURES, KERNEL_SIZE, bias=False) layer2 = BatchNorm2d(OUT_FEATURES) model = TestModel2(layer, layer2, None, "tanh") From d5cce9df0b5bff1f7643909cfcd7ce0ceb2d113b Mon Sep 17 00:00:00 2001 From: nroope Date: Fri, 12 Jun 2026 15:42:15 +0200 Subject: [PATCH 10/22] initial tracing of model (#42) Add torch.fx tracing that can be used when replacing layers with compressed variants, or by calling the function directly --- src/pquant/__init__.py | 14 +- src/pquant/core/torch/convert_to_onnx.py | 49 +- src/pquant/core/torch/layers.py | 8 +- src/pquant/core/torch/tracing.py | 409 ++++++++++++++++ tests/test_torch_missing_quantizer_tracing.py | 444 ++++++++++++++++++ 5 files changed, 907 insertions(+), 17 deletions(-) create mode 100644 src/pquant/core/torch/tracing.py create mode 100644 tests/test_torch_missing_quantizer_tracing.py diff --git a/src/pquant/__init__.py b/src/pquant/__init__.py index c5b9b04..6c27f9d 100644 --- a/src/pquant/__init__.py +++ b/src/pquant/__init__.py @@ -19,7 +19,14 @@ pdp_config, wanda_config, ) - from .core.torch import activations, layers, optimizers, pruning_methods, quantizer + from .core.torch import ( + activations, + layers, + optimizers, + pruning_methods, + quantizer, + tracing, + ) from .core.torch.layers import ( add_compression_layers, apply_final_compression, @@ -29,9 +36,10 @@ load_torch_hgq_model, post_training_prune, ) + from .core.torch.tracing import check_quantization, print_quantization_check from .core.torch.train import train_model - _forwards = ["activations", "layers", "quantizer", "optimizers"] + _forwards = ["activations", "layers", "quantizer", "optimizers", "tracing"] for name in _forwards: mod = importlib.import_module(f".core.torch.{name}", package="pquant") @@ -57,6 +65,8 @@ _forwards.append("load_from_dictionary") _forwards.append("get_ebops") _forwards.append("load_torch_hgq_model") + _forwards.append("check_quantization") + _forwards.append("print_quantization_check") _forwards.append("PQConfig") __all__ = _forwards diff --git a/src/pquant/core/torch/convert_to_onnx.py b/src/pquant/core/torch/convert_to_onnx.py index 14a7156..a61aeaa 100644 --- a/src/pquant/core/torch/convert_to_onnx.py +++ b/src/pquant/core/torch/convert_to_onnx.py @@ -53,6 +53,7 @@ PQLayerNorm, PQMultiheadAttention, ) +from pquant.core.torch.quantizer import Quantizer # noqa: E402 # --------------------------------------------------------------------------- # QONNX Quant node @@ -1200,6 +1201,15 @@ def _emit_module( module, prefix, current, current, current, nodes, initializers, quant_fn, use_qonnx, store_integer_weights ) return out + if isinstance(module, Quantizer): + # Standalone quantizer (e.g. an auto-inserted missing quantizer or a + # constant-matrix quantizer): emit a single QDQ node from its k/i/f. + k, i, f = module.get_quantization_bits() + new_nodes, out = quant_fn( + prefix, current, module.round_mode, k, i, f, initializers, overflow_mode=getattr(module, "overflow", "SAT") + ) + nodes.extend(new_nodes) + return out raise TypeError(f"Unsupported module type for ONNX export: {type(module).__name__}") @@ -1481,6 +1491,8 @@ class _PQTracer(_fx.Tracer): PQAvgPool1d, PQAvgPool2d, PQMultiheadAttention, + PQActivation, + Quantizer, ) def is_leaf_module(self, m: nn.Module, qualname: str) -> bool: @@ -1516,13 +1528,16 @@ def convert_to_onnx_fx( # need to expand torch's two-arg .transpose(d0, d1) into a full ONNX perm. from torch.fx.passes.shape_prop import ShapeProp + # Build the probe tensor on the model's own device so ShapeProp doesn't hit a + # device mismatch when a default device (e.g. CUDA) is set via torch.set_default_device. + device = next((p.device for p in model.parameters()), None) with torch.no_grad(): - ShapeProp(gm).propagate(torch.zeros(1, *input_shape)) + ShapeProp(gm).propagate(torch.zeros(1, *input_shape, device=device)) onnx_nodes: list[onnx.NodeProto] = [] initializers: list[onnx.TensorProto] = [] node_to_name: dict[_fx.Node, str] = {} - output_name: str = "" + output_names: list[str] = [] def _res(arg) -> str: if isinstance(arg, _fx.Node): @@ -1680,6 +1695,11 @@ def _resolve_perm_dims(args, rank: int) -> list[int]: onnx_nodes.append(oh.make_node("Relu", inputs=[_res(node.args[0])], outputs=[out])) node_to_name[node] = out + elif fn in (_F.sigmoid, torch.sigmoid): + out = f"{node.name}_sigmoid" + onnx_nodes.append(oh.make_node("Sigmoid", inputs=[_res(node.args[0])], outputs=[out])) + node_to_name[node] = out + elif fn is torch.flatten: start_dim = node.args[1] if len(node.args) > 1 else node.kwargs.get("start_dim", 0) out = f"{node.name}_flatten" @@ -1739,28 +1759,29 @@ def _resolve_perm_dims(args, rank: int) -> list[int]: elif node.op == "output": ret = node.args[0] - if isinstance(ret, _fx.Node): - val = node_to_name[ret] + rets = list(ret) if isinstance(ret, (tuple, list)) else [ret] + for r in rets: + if not isinstance(r, _fx.Node): + raise TypeError("FX ONNX export: unsupported (non-tensor) model output") + val = node_to_name[r] # MHA nodes store a tuple (out, avg_attn); expose the attention output. - output_name = val[0] if isinstance(val, tuple) else val - elif isinstance(ret, (tuple, list)) and len(ret) == 1: - val = node_to_name[ret[0]] - output_name = val[0] if isinstance(val, tuple) else val - else: - raise TypeError("Only single-output models are supported for FX ONNX export") + output_names.append(val[0] if isinstance(val, tuple) else val) with torch.no_grad(): - dummy_out = model(torch.zeros(1, *input_shape)) - output_shape = [None] + list(dummy_out.shape[1:]) + dummy_out = model(torch.zeros(1, *input_shape, device=device)) + dummy_outs = list(dummy_out) if isinstance(dummy_out, (tuple, list)) else [dummy_out] batch_dim = oh.make_tensor_value_info("input", TensorProto.FLOAT, [None, *input_shape]) - output_vi = oh.make_tensor_value_info(output_name, TensorProto.FLOAT, output_shape) + output_vis = [ + oh.make_tensor_value_info(name, TensorProto.FLOAT, [None] + list(t.shape[1:])) + for name, t in zip(output_names, dummy_outs) + ] onnx_graph = oh.make_graph( nodes=onnx_nodes, name="pquant_onnx_fx", inputs=[batch_dim], - outputs=[output_vi], + outputs=output_vis, initializer=initializers, ) diff --git a/src/pquant/core/torch/layers.py b/src/pquant/core/torch/layers.py index 3357185..7964172 100644 --- a/src/pquant/core/torch/layers.py +++ b/src/pquant/core/torch/layers.py @@ -644,9 +644,15 @@ def extra_repr(self): return s.format(**self.__dict__) -def add_compression_layers(model, config, input_shape=None): +def add_compression_layers(model, config, input_shape=None, add_missing_quantizers=False): model = add_quantized_activations_to_model_layer(model, config) model = add_pruning_to_model(model, config) + if add_missing_quantizers: + # Imported here (not at module top) to avoid a circular import: tracing.py + # imports the layer classes defined in this module. + from pquant.core.torch.tracing import check_quantization + + model = check_quantization(model, add_missing_quantizers=True, config=config) model.to("cuda") if input_shape is not None: model(torch.rand(input_shape).to("cuda")) diff --git a/src/pquant/core/torch/tracing.py b/src/pquant/core/torch/tracing.py new file mode 100644 index 0000000..0bbc30e --- /dev/null +++ b/src/pquant/core/torch/tracing.py @@ -0,0 +1,409 @@ +import logging + +import torch +import torch.nn.functional as F + +from pquant.core.torch.activations import PQActivation +from pquant.core.torch.layers import ( + PQAvgPoolBase, + PQBatchNorm1d, + PQBatchNorm2d, + PQWeightBiasBase, +) +from pquant.core.torch.quantizer import Quantizer + +_PQUANTML_LAYER_TYPES = (PQWeightBiasBase, PQAvgPoolBase, PQBatchNorm1d, PQBatchNorm2d, PQActivation) + + +def _analyze_quantization(model): + """Trace model with torch.fx and return (traced, node_issues, edges_to_quantize). + + node_issues maps each problematic Node to a list of human-readable + descriptions of what quantization is missing at that node. + + edges_to_quantize is a set of (producer_node, consumer_node) tuples + identifying graph edges on which a data quantizer should be inserted to + fix the missing quantization. + """ + import operator + from collections import defaultdict + + from torch.fx import GraphModule, Tracer + + class _PQTracer(Tracer): + def is_leaf_module(self, m, module_qualified_name): + if isinstance(m, _PQUANTML_LAYER_TYPES) or isinstance(m, Quantizer): + return True + return super().is_leaf_module(m, module_qualified_name) + + tracer = _PQTracer() + graph = tracer.trace(model) + traced = GraphModule(tracer.root, graph) + modules = dict(traced.named_modules()) + + arith_functions = { + operator.add, + operator.iadd, + operator.sub, + operator.isub, + operator.mul, + operator.imul, + operator.matmul, + operator.imatmul, + operator.truediv, + operator.itruediv, + operator.floordiv, + operator.ifloordiv, + operator.pow, + operator.ipow, + torch.add, + torch.sub, + torch.mul, + torch.div, + torch.divide, + torch.true_divide, + torch.floor_divide, + torch.matmul, + torch.bmm, + torch.mm, + torch.einsum, + torch.pow, + torch.cat, + torch.stack, + } + for _name in ("concat", "concatenate"): + _fn = getattr(torch, _name, None) + if _fn is not None: + arith_functions.add(_fn) + + _nonlin_torch_names = ( + "sigmoid", + "tanh", + "exp", + "log", + "log2", + "log10", + "sqrt", + "rsqrt", + "reciprocal", + "softmax", + "log_softmax", + "sin", + "cos", + "tan", + ) + _nonlin_F_names = ( + "sigmoid", + "tanh", + "softmax", + "log_softmax", + "gelu", + "silu", + "elu", + "selu", + "softplus", + "mish", + "hardsigmoid", + "hardswish", + "leaky_relu", + ) + nonlin_functions = set() + for _name in _nonlin_torch_names: + _fn = getattr(torch, _name, None) + if _fn is not None: + nonlin_functions.add(_fn) + for _name in _nonlin_F_names: + _fn = getattr(F, _name, None) + if _fn is not None: + nonlin_functions.add(_fn) + + quant_sensitive_functions = arith_functions | nonlin_functions + quant_sensitive_methods = { + "add", + "add_", + "__add__", + "__radd__", + "__iadd__", + "sub", + "sub_", + "__sub__", + "__rsub__", + "__isub__", + "mul", + "mul_", + "__mul__", + "__rmul__", + "__imul__", + "matmul", + "__matmul__", + "__rmatmul__", + "__imatmul__", + "div", + "div_", + "__truediv__", + "__rtruediv__", + "__itruediv__", + "floor_divide", + "floor_divide_", + "__floordiv__", + "__rfloordiv__", + "__ifloordiv__", + "true_divide", + "true_divide_", + "pow", + "pow_", + "__pow__", + "__rpow__", + "__ipow__", + "bmm", + "mm", + "einsum", + "sigmoid", + "sigmoid_", + "tanh", + "tanh_", + "exp", + "exp_", + "log", + "log_", + "log2", + "log2_", + "log10", + "log10_", + "sqrt", + "sqrt_", + "rsqrt", + "rsqrt_", + "reciprocal", + "reciprocal_", + "softmax", + "log_softmax", + "sin", + "sin_", + "cos", + "cos_", + "tan", + "tan_", + } + + def is_quant_sensitive(node): + if node.op == "call_function": + return node.target in quant_sensitive_functions + if node.op == "call_method": + return node.target in quant_sensitive_methods + return False + + def get_module(node): + if node.op == "call_module": + return modules.get(node.target) + return None + + quantized = {} + node_issues = defaultdict(list) + edges_to_quantize = set() + + for node in traced.graph.nodes: + input_nodes = node.all_input_nodes + all_inputs_quantized = bool(input_nodes) and all(quantized.get(n, False) for n in input_nodes) + + if node.op in ("placeholder", "get_attr"): + quantized[node] = False + elif node.op == "call_module": + mod = get_module(node) + if isinstance(mod, _PQUANTML_LAYER_TYPES): + if not getattr(mod, "quantize_input", False) and not all_inputs_quantized: + node_issues[node].append( + f"PQuantML layer '{node.target}' has quantize_input=False but receives unquantized input" + ) + for n in input_nodes: + if not quantized.get(n, False): + edges_to_quantize.add((n, node)) + # A relu activation is a grid-preserving clip (its optional multiplier is a + # power-of-two scale), so if the value reaching it is already quantized the + # output stays on-grid and needs no output quantizer. + grid_preserving = isinstance(mod, PQActivation) and mod.activation_name == "relu" + input_quantized = bool(getattr(mod, "quantize_input", False)) or all_inputs_quantized + quantized[node] = bool(getattr(mod, "quantize_output", False)) or (grid_preserving and input_quantized) + elif isinstance(mod, Quantizer): + quantized[node] = True + else: + quantized[node] = all_inputs_quantized + elif node.op in ("call_function", "call_method"): + if is_quant_sensitive(node): + for n in input_nodes: + if not quantized.get(n, False): + node_issues[node].append(f"input '{n.name}' is not quantized") + edges_to_quantize.add((n, node)) + quantized[node] = False + else: + quantized[node] = all_inputs_quantized + elif node.op == "output": + for n in input_nodes: + if not quantized.get(n, False): + node_issues[node].append(f"model output '{n.name}' is not quantized") + edges_to_quantize.add((n, node)) + quantized[node] = all_inputs_quantized + else: + quantized[node] = False + + return traced, node_issues, edges_to_quantize + + +def _insert_missing_quantizers(traced, edges_to_quantize, config): + from collections import defaultdict + + qp = config.quantization_parameters + + def _make_quantizer(): + return Quantizer( + k=qp.default_data_keep_negatives, + i=qp.default_data_integer_bits, + f=qp.default_data_fractional_bits, + overflow=qp.overflow_mode_data, + round_mode=qp.round_mode, + is_heterogeneous=False, + is_data=True, + granularity="per_tensor", + hgq_gamma=qp.hgq_gamma, + ) + + def _enable_pquantml_output_quantization(layer): + layer.quantize_output = True + if isinstance(layer, PQWeightBiasBase) and getattr(layer, "built", False) and not hasattr(layer, "output_quantizer"): + device = next(layer.parameters()).device + layer.output_quantizer = Quantizer( + k=torch.tensor(layer.k_output), + i=torch.tensor(layer.i_output), + f=torch.tensor(layer.f_output), + overflow=layer.overflow_mode_data, + round_mode=layer.round_mode, + is_heterogeneous=layer.use_hgq, + is_data=True, + hgq_gamma=layer.hgq_gamma, + place="datalane", + ).to(device) + + modules = dict(traced.named_modules()) + + by_producer = defaultdict(list) + for producer, consumer in edges_to_quantize: + by_producer[producer].append(consumer) + + graph = traced.graph + idx = 0 + for producer, consumers in by_producer.items(): + pqml_layer = None + if producer.op == "call_module": + mod = modules.get(producer.target) + if isinstance(mod, _PQUANTML_LAYER_TYPES): + pqml_layer = mod + if pqml_layer is not None: + _enable_pquantml_output_quantization(pqml_layer) + continue + q_name = f"_auto_missing_quantizer_{idx}" + idx += 1 + traced.add_module(q_name, _make_quantizer()) + with graph.inserting_after(producer): + qnode = graph.call_module(q_name, (producer,)) + for consumer in consumers: + consumer.replace_input_with(producer, qnode) + + graph.lint() + traced.recompile() + return traced + + +def check_quantization(model, add_missing_quantizers=False, config=None): + """Verify quantization is present everywhere in the model's forward graph. + + The model's input is assumed to be unquantized. The model is traced with + torch.fx; PQuantML layers and Quantizer modules are treated as leaves. + For each node the output's quantization state is propagated forward: + - PQuantML layers (PQWeightBiasBase, PQAvgPoolBase, PQBatchNorm1d, + PQBatchNorm2d, PQActivation): output is quantized iff quantize_output + is True. If quantize_input is False and the incoming data is not + already quantized, that is reported as missing quantization. + - Quantizer modules always produce quantized output. + - Non-PQuantML quant-sensitive ops β€” arithmetic/combining + (add/sub/mul/div/matmul/bmm/mm/einsum/cat/stack/pow) and off-grid + nonlinearities (sigmoid/tanh/softmax/exp/log/sqrt/rsqrt/reciprocal/ + gelu/silu/elu/selu/softplus/mish/hardsigmoid/hardswish/leaky_relu/ + sin/cos/tan), whether used as torch functions, torch.nn.functional + functions, operator.* functions, or tensor methods β€” require each + input to already be quantized; their own output is marked unquantized + so the next consumer's per-edge input check flags it if needed. + - The model's output node(s) are required to be quantized: any + unquantized return value is flagged. + - Other ops (shape-only views, indexing, non-arithmetic torch calls) + propagate the quantization state of their inputs. + + When `add_missing_quantizers` is False (default), returns True if every + location in the graph has the required quantization, otherwise a list of + strings describing each missing quantization. + + When `add_missing_quantizers` is True, `config` must be provided. For + every producer whose output feeds an edge with missing quantization: + - if the producer is a PQuantML layer, its `quantize_output` flag is + set to True (and, for built `PQWeightBiasBase` subclasses that lacked + an output_quantizer, one is constructed in-place from the layer's + own k_output/i_output/f_output and mode settings); + - otherwise, a fresh `Quantizer` module is instantiated with the + config's default data k/i/f and data round/overflow modes, attached + to the transformed module as `_auto_missing_quantizer_`, and a + `call_module` to that new quantizer is inserted on each affected + edge. Using one Quantizer per insertion site (rather than sharing + one) allows heterogeneous quantization to specialize per site. + Returns the transformed torch.fx GraphModule. + """ + traced, node_issues, edges_to_quantize = _analyze_quantization(model) + + if add_missing_quantizers: + if config is None: + raise ValueError("check_quantization(add_missing_quantizers=True) requires config") + if edges_to_quantize: + _insert_missing_quantizers(traced, edges_to_quantize, config) + return traced + + if not node_issues: + return True + messages = [] + for node, msgs in node_issues.items(): + for msg in msgs: + messages.append(f"'{node.name}' ({node.op}): {msg}") + return messages + + +def print_quantization_check(model, use_color=True): + """Print the traced model's graph with missing-quantization nodes flagged. + + Runs `check_quantization(model)` and prints the torch.fx graph line by + line. Nodes with missing quantization are highlighted in red (git-diff + style) and followed by one annotated line per issue. + + Set use_color=False to disable ANSI escape codes (e.g. when redirecting + to a file or a terminal that does not support colors). + + Returns the same value as `check_quantization(model)`. + """ + traced, node_issues, _ = _analyze_quantization(model) + + RED = "\033[31m" if use_color else "" + BOLD = "\033[1m" if use_color else "" + RESET = "\033[0m" if use_color else "" + + for node in traced.graph.nodes: + line = node.format_node() or f"{node.name} = {node.op} {node.target}" + if node in node_issues: + logging.info(f"{RED}{BOLD}- {line}{RESET}") + for msg in node_issues[node]: + logging.info(f"{RED} ! {msg}{RESET}") + else: + logging.info(f" {line}") + + if not node_issues: + return True + messages = [] + for node, msgs in node_issues.items(): + for msg in msgs: + messages.append(f"'{node.name}' ({node.op}): {msg}") + return messages diff --git a/tests/test_torch_missing_quantizer_tracing.py b/tests/test_torch_missing_quantizer_tracing.py new file mode 100644 index 0000000..60edf4f --- /dev/null +++ b/tests/test_torch_missing_quantizer_tracing.py @@ -0,0 +1,444 @@ +import os + +import pytest +import torch +from torch import nn + +os.environ["KERAS_BACKEND"] = "torch" + +from pquant.activations import PQActivation # noqa: E402 +from pquant.core.hyperparameter_optimization import PQConfig # noqa: E402 +from pquant.core.torch.quantizer import Quantizer # noqa: E402 +from pquant.core.torch.tracing import check_quantization # noqa: E402 +from pquant.layers import PQDense # noqa: E402 + +BATCH_SIZE = 4 +OUT_FEATURES = 32 +IN_FEATURES = 16 + + +@pytest.fixture +def config_pdp(): + cfg = { + "pruning_parameters": { + "disable_pruning_for_layers": [], + "enable_pruning": True, + "epsilon": 1.0, + "pruning_method": "pdp", + "sparsity": 0.75, + "temperature": 1e-5, + "threshold_decay": 0.0, + "structured_pruning": False, + }, + "quantization_parameters": { + "default_weight_integer_bits": 0.0, + "default_weight_fractional_bits": 7.0, + "default_data_integer_bits": 0.0, + "default_data_fractional_bits": 7.0, + "default_data_keep_negatives": 0.0, + "default_weight_keep_negatives": 1.0, + "quantize_input": True, + "quantize_output": False, + "enable_quantization": True, + "hgq_gamma": 0.0003, + "hgq_beta": 1e-5, + "hgq_heterogeneous": True, + "layer_specific": {}, + "use_high_granularity_quantization": False, + "use_real_tanh": False, + "use_relu_multiplier": True, + "use_symmetric_quantization": False, + "round_mode": "RND", + "overflow_mode_parameters": "SAT", + "overflow_mode_data": "SAT", + "granularity": "per_tensor", + }, + "training_parameters": {"pruning_first": False}, + "fitcompress_parameters": {"enable_fitcompress": False}, + } + return PQConfig.load_from_config(cfg) + + +def test_check_quantization_passes(config_pdp): + class GoodModel(nn.Module): + def __init__(self): + super().__init__() + self.d1 = PQDense(config_pdp, IN_FEATURES, OUT_FEATURES, quantize_input=True, quantize_output=True) + self.act = PQActivation(config_pdp, "relu", quantize_input=False, quantize_output=True) + self.d2 = PQDense(config_pdp, OUT_FEATURES, OUT_FEATURES, quantize_input=False, quantize_output=True) + + def forward(self, x): + return self.d2(self.act(self.d1(x))) + + assert check_quantization(GoodModel()) is True + + +def test_check_quantization_fails(config_pdp): + # d1.quantize_output=False, so the `a + a` add receives unquantized inputs. + class BadModel(nn.Module): + def __init__(self): + super().__init__() + self.d1 = PQDense(config_pdp, IN_FEATURES, OUT_FEATURES, quantize_input=True, quantize_output=False) + self.d2 = PQDense(config_pdp, OUT_FEATURES, OUT_FEATURES, quantize_input=False, quantize_output=True) + + def forward(self, x): + a = self.d1(x) + b = a + a + return self.d2(b) + + result = check_quantization(BadModel()) + assert isinstance(result, list) + assert len(result) >= 1 + joined = "\n".join(result) + assert "add" in joined + assert "d2" in joined + assert "not quantized" in joined + + +def _build_chain_model(config, a_qin, a_qout, b_qin, b_qout, op="add"): + class ChainModel(nn.Module): + def __init__(self): + super().__init__() + self.a = PQDense(config, IN_FEATURES, OUT_FEATURES, quantize_input=a_qin, quantize_output=a_qout) + self.b = PQDense(config, OUT_FEATURES, OUT_FEATURES, quantize_input=b_qin, quantize_output=b_qout) + + def forward(self, x): + y = self.a(x) + if op == "add": + y = y + y + elif op == "sigmoid": + y = torch.sigmoid(y) + else: + raise ValueError(op) + return self.b(y) + + return ChainModel() + + +def test_check_quantization_chain_all_true_passes(config_pdp): + model = _build_chain_model(config_pdp, True, True, True, True) + assert check_quantization(model) is True + + +def test_check_quantization_chain_a_qin_false(config_pdp): + # 'a' has quantize_input=False but receives the raw (unquantized) model input. + model = _build_chain_model(config_pdp, False, True, True, True) + result = check_quantization(model) + assert isinstance(result, list) + joined = "\n".join(result) + assert "'a'" in joined + assert "quantize_input=False" in joined + assert "'b'" not in joined + assert "add" not in joined + + +def test_check_quantization_chain_b_qin_false(config_pdp): + # 'b' has quantize_input=False but its input comes from the unquantized `add` op output. + model = _build_chain_model(config_pdp, True, True, True, True) + model.b.quantize_input = False + result = check_quantization(model) + assert isinstance(result, list) + joined = "\n".join(result) + assert "'b'" in joined + assert "quantize_input=False" in joined + assert "'a'" not in joined + + +def test_check_quantization_chain_a_qout_false(config_pdp): + # 'a'.quantize_output=False, so the `add` op consuming 'a' receives unquantized input. + model = _build_chain_model(config_pdp, True, False, True, True) + result = check_quantization(model) + assert isinstance(result, list) + joined = "\n".join(result) + assert "'add'" in joined + assert "'a'" in joined + assert "not quantized" in joined + assert "'b'" not in joined + + +def test_check_quantization_chain_b_qout_false(config_pdp): + # 'b'.quantize_output=False, so the model output is left unquantized. + model = _build_chain_model(config_pdp, True, True, True, False) + result = check_quantization(model) + assert isinstance(result, list) + joined = "\n".join(result) + assert "'output'" in joined + assert "'b'" in joined + assert "not quantized" in joined + + +def test_check_quantization_chain_both_qin_false(config_pdp): + # Both 'a' and 'b' have quantize_input=False while their inputs are unquantized. + model = _build_chain_model(config_pdp, False, True, False, True) + result = check_quantization(model) + assert isinstance(result, list) + joined = "\n".join(result) + assert "'a'" in joined + assert "'b'" in joined + assert joined.count("quantize_input=False") == 2 + + +def test_check_quantization_chain_both_qout_false(config_pdp): + # First layer output and model output unquantized + model = _build_chain_model(config_pdp, True, False, True, False) + result = check_quantization(model) + assert isinstance(result, list) + joined = "\n".join(result) + assert "'add'" in joined + assert "'output'" in joined + assert joined.count("not quantized") == 2 + + +def test_check_quantization_chain_all_false(config_pdp): + # All quantize_input/output flags are False. + model = _build_chain_model(config_pdp, False, False, False, False) + result = check_quantization(model) + assert isinstance(result, list) + joined = "\n".join(result) + assert "'a'" in joined + assert "'add'" in joined + assert "'b'" in joined + assert "'output'" in joined + assert joined.count("quantize_input=False") == 2 + + +def test_check_quantization_unary_op_between(config_pdp): + # 'a'.quantize_output=False. + model = _build_chain_model(config_pdp, True, False, True, True, op="sigmoid") + result = check_quantization(model) + assert isinstance(result, list) + joined = "\n".join(result) + assert "sigmoid" in joined + assert "'a'" in joined + + +def test_check_quantization_fix_pqml_producer_flips_flag_unbuilt(config_pdp): + # 'a'.quantize_output=False feeds the `add` op, change quantize_output to True. + model = _build_chain_model(config_pdp, True, False, True, True) + assert model.a.quantize_output is False + traced = check_quantization(model, add_missing_quantizers=True, config=config_pdp) + assert model.a.quantize_output is True + added = [name for name, _ in traced.named_modules() if name.startswith("_auto_missing_quantizer_")] + assert added == [] + assert check_quantization(traced) is True + + +def test_check_quantization_fix_pqml_producer_flips_flag_built(config_pdp): + # Same as above but 'a' is already built: the fix flips the flag and constructs its output_quantizer in-place. + model = _build_chain_model(config_pdp, True, False, True, True) + x = torch.randn(BATCH_SIZE, IN_FEATURES) + model(x) + assert model.a.built is True + assert not hasattr(model.a, "output_quantizer") + traced = check_quantization(model, add_missing_quantizers=True, config=config_pdp) + assert model.a.quantize_output is True + assert hasattr(model.a, "output_quantizer") + added = [name for name, _ in traced.named_modules() if name.startswith("_auto_missing_quantizer_")] + assert added == [] + assert check_quantization(traced) is True + + +def test_check_quantization_fix_output_flips_pqml_flag(config_pdp): + # 'b'.quantize_output=False leaves the model output unquantized, so the fix flips 'b'.quantize_output. + model = _build_chain_model(config_pdp, True, True, True, False) + traced = check_quantization(model, add_missing_quantizers=True, config=config_pdp) + assert model.b.quantize_output is True + added = [name for name, _ in traced.named_modules() if name.startswith("_auto_missing_quantizer_")] + assert added == [] + assert check_quantization(traced) is True + + +def test_check_quantization_fix_placeholder_producer(config_pdp): + # `x + x` consumes the raw placeholder input (unquantized), so the fix inserts a standalone quantizer. + class PHModel(nn.Module): + def __init__(self): + super().__init__() + self.b = PQDense(config_pdp, IN_FEATURES, OUT_FEATURES, quantize_input=True, quantize_output=True) + + def forward(self, x): + return self.b(x + x) + + model = PHModel() + traced = check_quantization(model, add_missing_quantizers=True, config=config_pdp) + added = [name for name, _ in traced.named_modules() if name.startswith("_auto_missing_quantizer_")] + assert len(added) == 1 + assert check_quantization(traced) is True + + +def test_check_quantization_fix_functional_producer_inserts_quantizer(config_pdp): + # `add` feeds `sigmoid` and `sigmoid` feeds 'b' (both function outputs are unquantized). + class ChainedOps(nn.Module): + def __init__(self): + super().__init__() + self.a = PQDense(config_pdp, IN_FEATURES, OUT_FEATURES, quantize_input=True, quantize_output=True) + self.b = PQDense(config_pdp, OUT_FEATURES, OUT_FEATURES, quantize_input=False, quantize_output=True) + + def forward(self, x): + y = self.a(x) + y = torch.sigmoid(y + y) + return self.b(y) + + model = ChainedOps() + traced = check_quantization(model, add_missing_quantizers=True, config=config_pdp) + added = [name for name, _ in traced.named_modules() if name.startswith("_auto_missing_quantizer_")] + assert len(added) == 2 + assert check_quantization(traced) is True + + +def test_check_quantization_fix_roundtrip_all_false(config_pdp): + # All flags False, so the fix must flip both layers' quantize_output to fully quantize the graph. + model = _build_chain_model(config_pdp, False, False, False, False) + traced = check_quantization(model, add_missing_quantizers=True, config=config_pdp) + assert model.a.quantize_output is True + assert model.b.quantize_output is True + assert check_quantization(traced) is True + + +def test_check_quantization_fix_requires_config(config_pdp): + # Fixing missing quantizer requires a config (here omitted, so it raises an error). + model = _build_chain_model(config_pdp, True, False, True, True) + with pytest.raises(ValueError): + check_quantization(model, add_missing_quantizers=True) + + +def test_check_quantization_pqactivation_relu_preserves_quantization(config_pdp): + # A relu PQActivation with quantize_input=True quantizes its input, and relu is a + # grid-preserving clip, so its output is already quantized: the `add` it feeds needs + # no extra quantizer. The model passes as-is and the activation's quantize_output stays False. + class ActChain(nn.Module): + def __init__(self): + super().__init__() + self.act = PQActivation(config_pdp, "relu", quantize_input=True, quantize_output=False) + self.b = PQDense(config_pdp, IN_FEATURES, OUT_FEATURES, quantize_input=True, quantize_output=True) + + def forward(self, x): + y = self.act(x) + y = y + y + return self.b(y) + + model = ActChain() + assert check_quantization(model) is True + traced = check_quantization(model, add_missing_quantizers=True, config=config_pdp) + assert model.act.quantize_output is False # relu needs no output quantizer + added = [name for name, _ in traced.named_modules() if name.startswith("_auto_missing_quantizer_")] + assert added == [] + assert check_quantization(traced) is True + + +def test_check_quantization_pqactivation_nonlinear_producer_fix(config_pdp): + # A tanh PQActivation is NOT grid-preserving, so its quantize_output=False output feeds + # the `add` unquantized and the fix flips the activation's quantize_output to True. + class ActChain(nn.Module): + def __init__(self): + super().__init__() + self.act = PQActivation(config_pdp, "tanh", quantize_input=True, quantize_output=False) + self.b = PQDense(config_pdp, IN_FEATURES, OUT_FEATURES, quantize_input=True, quantize_output=True) + + def forward(self, x): + y = self.act(x) + y = y + y + return self.b(y) + + model = ActChain() + result = check_quantization(model) + assert isinstance(result, list) + traced = check_quantization(model, add_missing_quantizers=True, config=config_pdp) + assert model.act.quantize_output is True + assert check_quantization(traced) is True + + +def test_check_quantization_multiple_consumers_of_pqml_output(config_pdp): + # 'a'.quantize_output=False feeds both the `add` op and the model output, quantize a output. + class MultiConsumer(nn.Module): + def __init__(self): + super().__init__() + self.a = PQDense(config_pdp, IN_FEATURES, OUT_FEATURES, quantize_input=True, quantize_output=False) + self.b = PQDense(config_pdp, OUT_FEATURES, OUT_FEATURES, quantize_input=True, quantize_output=True) + + def forward(self, x): + y = self.a(x) + z = y + y + return self.b(z), y + + model = MultiConsumer() + result = check_quantization(model) + assert isinstance(result, list) + joined = "\n".join(result) + assert "'add'" in joined + assert "'output'" in joined + traced = check_quantization(model, add_missing_quantizers=True, config=config_pdp) + assert model.a.quantize_output is True + added = [name for name, _ in traced.named_modules() if name.startswith("_auto_missing_quantizer_")] + assert added == [] + assert check_quantization(traced) is True + + +def _make_quantizer(config): + qp = config.quantization_parameters + return Quantizer( + k=qp.default_data_keep_negatives, + i=qp.default_data_integer_bits, + f=qp.default_data_fractional_bits, + overflow=qp.overflow_mode_data, + round_mode=qp.round_mode, + is_heterogeneous=False, + is_data=True, + granularity="per_tensor", + hgq_gamma=qp.hgq_gamma, + ) + + +def _build_dense_matmul_skip_model(config, dense_qout): + class DenseMatmulSkip(nn.Module): + def __init__(self): + super().__init__() + self.d = PQDense(config, IN_FEATURES, IN_FEATURES, quantize_input=True, quantize_output=dense_qout) + self.register_buffer("w", torch.randn(IN_FEATURES, IN_FEATURES)) + # Route the constant matrix through a Quantizer so it counts as + # "assumed quantized" (the tracer treats a raw get_attr as unquantized). + self.wq = _make_quantizer(config) + + def forward(self, x): + y = self.d(x) + y = torch.matmul(y, self.wq(self.w)) # matmul with constant matrix (assumed quantized) + y = y + x # skip connection from the (unquantized) model input + return y + + return DenseMatmulSkip() + + +def test_check_quantization_dense_matmul_skip_from_input(config_pdp): + # matmul output is unquantized and the skip connection feeds the raw model input 'x' into the + # `add`, so the add inputs and the model output are flagged; the assumed-quantized constant is not. + model = _build_dense_matmul_skip_model(config_pdp, dense_qout=True) + result = check_quantization(model) + assert isinstance(result, list) + joined = "\n".join(result) + assert "'add'" in joined + assert "input 'x' is not quantized" in joined + assert "input 'matmul' is not quantized" in joined + assert "'output'" in joined + # The constant matmul operand is quantized, so matmul itself reports no missing input. + assert not any(r.startswith("'matmul'") for r in result) + + +def test_check_quantization_dense_matmul_skip_from_input_fix(config_pdp): + # Fixing inserts standalone quantizers on the unquantized skip input, the matmul output, and the + # model output (the assumed-quantized constant needs none). + model = _build_dense_matmul_skip_model(config_pdp, dense_qout=True) + traced = check_quantization(model, add_missing_quantizers=True, config=config_pdp) + added = [name for name, _ in traced.named_modules() if name.startswith("_auto_missing_quantizer_")] + assert len(added) == 3 + assert check_quantization(traced) is True + + +def test_check_quantization_dense_matmul_skip_unquantized_dense(config_pdp): + # With dense.quantize_output=False the matmul also sees an unquantized data input from 'd'; the fix + # flips the PQDense flag (a PQuantML producer) and inserts quantizers for the rest. + model = _build_dense_matmul_skip_model(config_pdp, dense_qout=False) + result = check_quantization(model) + assert isinstance(result, list) + assert any(r.startswith("'matmul'") and "input 'd' is not quantized" in r for r in result) + traced = check_quantization(model, add_missing_quantizers=True, config=config_pdp) + assert model.d.quantize_output is True + assert check_quantization(traced) is True From 31a856d9e3d4b3c09d065f019006f9bd7cead977 Mon Sep 17 00:00:00 2001 From: Roope Niemi Date: Fri, 12 Jun 2026 17:56:02 +0200 Subject: [PATCH 11/22] rtl_predict in alkaid converter tests --- tests/test_keras_alkaid_conversion.py | 45 ++++++++++++--------------- tests/test_torch_alkaid_conversion.py | 41 +++++++++++------------- 2 files changed, 39 insertions(+), 47 deletions(-) diff --git a/tests/test_keras_alkaid_conversion.py b/tests/test_keras_alkaid_conversion.py index 4c988fc..b96c185 100644 --- a/tests/test_keras_alkaid_conversion.py +++ b/tests/test_keras_alkaid_conversion.py @@ -63,6 +63,18 @@ def _build_model(config): return keras.Model([img_in, seq_in], x) +def _rtl_predict(comb, path, data): + """Write the RTL project, compile the simulation emulator, and run bit-accurate inference.""" + rtl_model = RTLModel(comb, str(path), "model", flavor="verilog", latency_cutoff=5, clock_period=5.0, print_latency=False) + rtl_model.write() + rtl_model.compile() + if isinstance(data, list): + data = [a.astype(np.float64) for a in data] + else: + data = data.astype(np.float64) + return rtl_model.predict(data) + + def _random_prune(layer, fraction, rng): """Zero exactly ``fraction`` of the layer's weights via its pruning mask.""" mask = layer.pruning_layer.mask @@ -123,20 +135,12 @@ def test_alkaid_rtl_matches_model(tmp_path): seq = rng.integers(0, 16, size=(n_samples,) + SEQ_SHAPE).astype("float32") / 16.0 reference = np.asarray(model([img, seq]), dtype=np.float64) # (n_samples, OUT_FEATURES) - emulated = np.stack( - [ - np.asarray(comb(np.concatenate([img[n].ravel(), seq[n].ravel()]), quantize=True), dtype=np.float64) - for n in range(n_samples) - ] - ) + emulated = _rtl_predict(comb, tmp_path, [img, seq]) + assert (tmp_path / "src" / "model.v").exists() assert np.any(reference != 0) # the comparison is non-trivial np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) - # Generate the actual RTL project from the same combinational logic. - RTLModel(comb, str(tmp_path), "model", flavor="verilog", print_latency=False).write() - assert (tmp_path / "src" / "model.v").exists() - # --- Coverage of every PQ layer the keras Alkaid plugin handles --------------- @@ -231,19 +235,12 @@ def test_alkaid_conversion_all_layer_types(tmp_path): img = rng.integers(0, 16, size=(n_samples,) + ALL_IMG_SHAPE).astype("float32") / 16.0 seq = rng.integers(0, 16, size=(n_samples,) + ALL_SEQ_SHAPE).astype("float32") / 16.0 reference = np.asarray(model([img, seq]), dtype=np.float64) - emulated = np.stack( - [ - np.asarray(comb(np.concatenate([img[n].ravel(), seq[n].ravel()]), quantize=True), dtype=np.float64) - for n in range(n_samples) - ] - ) + emulated = _rtl_predict(comb, tmp_path, [img, seq]) + assert (tmp_path / "src" / "model.v").exists() assert np.any(reference != 0) np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) - RTLModel(comb, str(tmp_path), "model", flavor="verilog", print_latency=False).write() - assert (tmp_path / "src" / "model.v").exists() - # --- Per-layer conversion: a model that is a single layer --------------------- @@ -313,7 +310,7 @@ def _single_layer_model(input_shape, layer, tail=None): @pytest.mark.parametrize("case_id", list(_SINGLE_LAYER_CASES)) -def test_alkaid_single_layer(case_id): +def test_alkaid_single_layer(case_id, tmp_path): config = pdp_config() config.quantization_parameters.enable_quantization = True input_shape, model = _SINGLE_LAYER_CASES[case_id](config) @@ -328,7 +325,7 @@ def test_alkaid_single_layer(case_id): n_samples = 16 x = rng.integers(0, 16, size=(n_samples,) + input_shape).astype("float32") / 16.0 reference = np.asarray(model(x), dtype=np.float64).reshape(n_samples, -1) - emulated = np.stack([np.asarray(comb(x[i].ravel(), quantize=True), dtype=np.float64) for i in range(n_samples)]) + emulated = _rtl_predict(comb, tmp_path, x) assert np.any(reference != 0) np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) @@ -370,10 +367,8 @@ def test_alkaid_multihead_attention(tmp_path): n_samples = 16 x = rng.integers(0, 16, size=(n_samples, MHA_SEQ_LEN, MHA_EMBED_DIM)).astype("float32") / 16.0 reference = np.asarray(model(x), dtype=np.float64).reshape(n_samples, -1) - emulated = np.stack([np.asarray(comb(x[i].ravel(), quantize=True), dtype=np.float64) for i in range(n_samples)]) + emulated = _rtl_predict(comb, tmp_path, x) + assert (tmp_path / "src" / "model.v").exists() assert np.any(reference != 0) np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) - - RTLModel(comb, str(tmp_path), "model", flavor="verilog", print_latency=False).write() - assert (tmp_path / "src" / "model.v").exists() diff --git a/tests/test_torch_alkaid_conversion.py b/tests/test_torch_alkaid_conversion.py index d62c94d..dfb0370 100644 --- a/tests/test_torch_alkaid_conversion.py +++ b/tests/test_torch_alkaid_conversion.py @@ -84,6 +84,18 @@ def _fixed_point_input(shape, kif=INPUT_KIF): return FVArray.from_kif(k, i, f, HWCONF, 0, None) +def _rtl_predict(comb, path, data): + """Write the RTL project, compile the simulation emulator, and run bit-accurate inference.""" + rtl_model = RTLModel(comb, str(path), "model", flavor="verilog", latency_cutoff=5, clock_period=5.0, print_latency=False) + rtl_model.write() + rtl_model.compile() + if isinstance(data, list): + data = [a.astype(np.float64) for a in data] + else: + data = data.astype(np.float64) + return rtl_model.predict(data) + + def _build_pruned_compressed_model(config, rng): """Build the model, build it (one forward), prune 90%, apply final compression, eval.""" model = TwoBranchNet(config) @@ -140,20 +152,12 @@ def test_alkaid_rtl_matches_model(tmp_path): model(torch.tensor(img, device=device), torch.tensor(seq, device=device)).cpu().numpy().astype(np.float64) ) # (n_samples, OUT_FEATURES) - emulated = np.stack( - [ - np.asarray(comb(np.concatenate([img[n].ravel(), seq[n].ravel()]), quantize=True), dtype=np.float64) - for n in range(n_samples) - ] - ) + emulated = _rtl_predict(comb, tmp_path, [img, seq]) + assert (tmp_path / "src" / "model.v").exists() assert np.any(reference != 0) # the comparison is non-trivial np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) - # Generate the actual RTL project from the same combinational logic. - RTLModel(comb, str(tmp_path), "model", flavor="verilog", print_latency=False).write() - assert (tmp_path / "src" / "model.v").exists() - # --- Coverage of every PQ layer the torch Alkaid plugin handles --------------- @@ -243,19 +247,12 @@ def test_alkaid_conversion_all_layer_types(tmp_path): reference = ( model(torch.tensor(img, device=device), torch.tensor(seq, device=device)).cpu().numpy().astype(np.float64) ) - emulated = np.stack( - [ - np.asarray(comb(np.concatenate([img[n].ravel(), seq[n].ravel()]), quantize=True), dtype=np.float64) - for n in range(n_samples) - ] - ) + emulated = _rtl_predict(comb, tmp_path, [img, seq]) + assert (tmp_path / "src" / "model.v").exists() assert np.any(reference != 0) np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) - RTLModel(comb, str(tmp_path), "model", flavor="verilog", print_latency=False).write() - assert (tmp_path / "src" / "model.v").exists() - # --- Per-layer conversion: a model that is a single layer --------------------- @@ -309,7 +306,7 @@ def forward(self, x): @pytest.mark.parametrize("case_id", list(_SINGLE_LAYER_CASES)) -def test_alkaid_single_layer(case_id): +def test_alkaid_single_layer(case_id, tmp_path): config = pdp_config() config.quantization_parameters.enable_quantization = True shape, model = _SINGLE_LAYER_CASES[case_id](config) @@ -328,7 +325,7 @@ def test_alkaid_single_layer(case_id): x = rng.integers(0, 16, size=(n_samples,) + shape[1:]).astype("float32") / 16.0 with torch.no_grad(): reference = model(torch.tensor(x)).cpu().numpy().reshape(n_samples, -1).astype(np.float64) - emulated = np.stack([np.asarray(comb(x[i].ravel(), quantize=True), dtype=np.float64) for i in range(n_samples)]) + emulated = _rtl_predict(comb, tmp_path, x) assert np.any(reference != 0) np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) @@ -384,7 +381,7 @@ def test_alkaid_multihead_attention(tmp_path): x = rng.integers(0, 16, size=(n_samples, MHA_SEQ_LEN, MHA_EMBED_DIM)).astype("float32") / 16.0 with torch.no_grad(): reference = model(torch.tensor(x)).cpu().numpy().reshape(n_samples, -1).astype(np.float64) - emulated = np.stack([np.asarray(comb(x[i].ravel(), quantize=True), dtype=np.float64) for i in range(n_samples)]) + emulated = _rtl_predict(comb, tmp_path, x) assert np.any(reference != 0) np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) From b859c2f7f1dd25834feba4e76e66b3ffb2dbfa26 Mon Sep 17 00:00:00 2001 From: Roope Niemi Date: Tue, 16 Jun 2026 12:58:12 +0200 Subject: [PATCH 12/22] fix bias per channel quantization bug --- src/pquant/core/torch/quantizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pquant/core/torch/quantizer.py b/src/pquant/core/torch/quantizer.py index 7f0a0e9..e2ff630 100644 --- a/src/pquant/core/torch/quantizer.py +++ b/src/pquant/core/torch/quantizer.py @@ -91,7 +91,7 @@ def compute_data_dynamic_bits(self, x): return self.calculate_bits_from_abs(abs_x) def compute_weight_dynamic_bits(self, x): - if self.granularity == "per_tensor": + if self.granularity == "per_tensor" or x.ndim == 1: _, i, f = self.get_quantization_bits() return i, f if self.granularity == "per_channel": From 0a30a3a189963df2cccb368fc4a642500c094b15 Mon Sep 17 00:00:00 2001 From: nroope Date: Thu, 9 Jul 2026 18:55:28 +0200 Subject: [PATCH 13/22] hotfix buggy keras quantizer b-variable initialization --- src/pquant/core/keras/quantizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pquant/core/keras/quantizer.py b/src/pquant/core/keras/quantizer.py index 20910aa..396c8ac 100644 --- a/src/pquant/core/keras/quantizer.py +++ b/src/pquant/core/keras/quantizer.py @@ -108,7 +108,7 @@ def build(self, input_shape): self.i = self.add_weight(shape=(), initializer=keras.initializers.Constant(self.i_init), trainable=False) self.f = self.add_weight(shape=(), initializer=keras.initializers.Constant(self.f_init), trainable=False) self.b = self.add_weight( - shape=(), initializer=keras.initializers.Constant(self.k_init + self.f_init + self.f_init), trainable=False + shape=(), initializer=keras.initializers.Constant(self.k_init + self.i_init + self.f_init), trainable=False ) else: i, _ = self.compute_dynamic_bits(keras.ops.ones(input_shape)) @@ -117,7 +117,7 @@ def build(self, input_shape): self.f = self.add_weight(shape=i.shape, initializer=keras.initializers.Constant(self.f_init), trainable=False) self.b = self.add_weight( shape=i.shape, - initializer=keras.initializers.Constant(self.k_init + self.f_init + self.f_init), + initializer=keras.initializers.Constant(self.k_init + self.i_init + self.f_init), trainable=False, ) From 14f3aef3a3cc52364e640978813161abbe06cfa6 Mon Sep 17 00:00:00 2001 From: nroope Date: Tue, 21 Jul 2026 16:05:26 +0200 Subject: [PATCH 14/22] cleanup alkaid converter (#50) * cleanup alkaid converter, tests, added final_compression_done hotfix for batchnorm, layernorm --- src/pquant/_alkaid_plugin/_alkaid_common.py | 111 +++++-- .../_alkaid_plugin/_alkaid_keras_plugin.py | 140 +++----- .../_alkaid_plugin/_alkaid_torch_plugin.py | 302 ++++++++---------- src/pquant/core/keras/layers.py | 2 +- src/pquant/core/torch/layers.py | 6 +- tests/test_keras_alkaid_conversion.py | 134 +++----- tests/test_torch_alkaid_conversion.py | 122 +++---- 7 files changed, 354 insertions(+), 463 deletions(-) diff --git a/src/pquant/_alkaid_plugin/_alkaid_common.py b/src/pquant/_alkaid_plugin/_alkaid_common.py index 687dba9..77b64e4 100644 --- a/src/pquant/_alkaid_plugin/_alkaid_common.py +++ b/src/pquant/_alkaid_plugin/_alkaid_common.py @@ -10,7 +10,7 @@ class PQuantAlkaidError(ValueError): """Raised for PQuant states that cannot be replayed by Alkaid.""" -def to_numpy(value: Any) -> np.ndarray: +def _to_numpy(value: Any) -> np.ndarray: if value is None: return np.array(0.0) if isinstance(value, np.ndarray): @@ -28,11 +28,11 @@ def to_numpy(value: Any) -> np.ndarray: return np.asarray(value) -def to_bool(value: Any, default: bool = False) -> bool: +def _to_bool(value: Any, default: bool = False) -> bool: if value is None: return default try: - arr = to_numpy(value) + arr = _to_numpy(value) except Exception: return bool(value) if arr.shape == (): @@ -40,11 +40,11 @@ def to_bool(value: Any, default: bool = False) -> bool: return bool(np.all(arr)) -def to_int_bits(value: Any) -> np.ndarray: - return np.rint(to_numpy(value)).astype(np.int64) +def _to_int_bits(value: Any) -> np.ndarray: + return np.rint(_to_numpy(value)).astype(np.int64) -def raw_module_attr(obj: Any, name: str, default: Any = None) -> Any: +def _raw_module_attr(obj: Any, name: str, default: Any = None) -> Any: for storage_name in ('_parameters', '_buffers', '_modules'): storage = getattr(obj, storage_name, None) if isinstance(storage, dict) and name in storage: @@ -55,38 +55,99 @@ def raw_module_attr(obj: Any, name: str, default: Any = None) -> Any: return getattr(obj, name, default) -def quantizer_kif(quantizer: Any) -> tuple[np.ndarray, np.ndarray, np.ndarray]: +def _quantizer_kif(quantizer: Any) -> tuple[np.ndarray, np.ndarray, np.ndarray]: if hasattr(quantizer, '_parameters'): - if not bool(raw_module_attr(quantizer, 'use_hgq', False)): + if not bool(_raw_module_attr(quantizer, 'use_hgq', False)): return ( - to_int_bits(raw_module_attr(quantizer, 'k')), - to_int_bits(raw_module_attr(quantizer, 'i')), - to_int_bits(raw_module_attr(quantizer, 'f')), + _to_int_bits(_raw_module_attr(quantizer, 'k')), + _to_int_bits(_raw_module_attr(quantizer, 'i')), + _to_int_bits(_raw_module_attr(quantizer, 'f')), ) - inner = raw_module_attr(quantizer, 'quantizer') + inner = _raw_module_attr(quantizer, 'quantizer') if hasattr(inner, '_parameters') or hasattr(inner, '_buffers'): - k = raw_module_attr(inner, '_k') - i = raw_module_attr(inner, '_i_raw', None) + k = _raw_module_attr(inner, '_k') + i = _raw_module_attr(inner, '_i_raw', None) if i is None: - i = raw_module_attr(inner, '_i') - f = raw_module_attr(inner, '_f') - return to_int_bits(k), to_int_bits(i), to_int_bits(f) + i = _raw_module_attr(inner, '_i') + f = _raw_module_attr(inner, '_f') + return _to_int_bits(k), _to_int_bits(i), _to_int_bits(f) k, i, f = quantizer.get_quantization_bits() - return to_int_bits(k), to_int_bits(i), to_int_bits(f) + return _to_int_bits(k), _to_int_bits(i), _to_int_bits(f) -def replay_quantizer(quantizer: Any, x: Any) -> Any: - k, i, f = quantizer_kif(quantizer) - inner = raw_module_attr(quantizer, 'quantizer', None) - overflow = raw_module_attr(quantizer, 'overflow', raw_module_attr(inner, 'overflow_mode', 'WRAP')) - round_mode = raw_module_attr(quantizer, 'round_mode', raw_module_attr(inner, 'round_mode', 'TRN')) +def _replay_quantizer(quantizer: Any, x: Any) -> Any: + k, i, f = _quantizer_kif(quantizer) + inner = _raw_module_attr(quantizer, 'quantizer', None) + overflow = _raw_module_attr(quantizer, 'overflow', _raw_module_attr(inner, 'overflow_mode', 'WRAP')) + round_mode = _raw_module_attr(quantizer, 'round_mode', _raw_module_attr(inner, 'round_mode', 'TRN')) return alkaid_quantize(x, k=k, i=i, f=f, overflow_mode=str(overflow).upper(), round_mode=str(round_mode).upper()) -def replay_quantizer_if_enabled(layer: Any, quantizer_name: str, x: Any, flag_name: str) -> Any: +def _replay_quantizer_if_enabled(layer: Any, quantizer_name: str, x: Any, flag_name: str) -> Any: if not bool(getattr(layer, 'enable_quantization', True)): return x if not bool(getattr(layer, flag_name, True)): return x quantizer = getattr(layer, quantizer_name, None) - return replay_quantizer(quantizer, x) + return _replay_quantizer(quantizer, x) + + +def _assert_final_compression(layer: Any) -> None: + if not _to_bool(_raw_module_attr(layer, 'final_compression_done', False)): + raise PQuantAlkaidError( + f'{type(layer).__name__} must have apply_final_compression() applied before Alkaid conversion.' + ) + + +def _final_bias(layer: Any) -> np.ndarray: + """The layer's final (compressed) bias as numpy, or a scalar zero when absent.""" + _assert_final_compression(layer) + bias = getattr(layer, '_bias', None) + if bias is None: + return np.array(0.0) + return _to_numpy(bias) + + +def _scale_by_relu_multiplier(layer: Any, x: Any) -> Any: + """Apply PQActivation's power-of-two ReLU multiplier, which is used only without HGQ.""" + applies = ( + not _to_bool(getattr(layer, 'use_hgq', False)) + and _to_bool(getattr(layer, 'use_multiplier', False)) + and layer.activation_name == 'relu' + and hasattr(layer, 'multiplier') + ) + if not applies: + return x + return x * (2.0 ** np.rint(_to_numpy(layer.multiplier))) + + +def _replay_table(table: Any, x: Any, table_fn: Any) -> Any: + """Replay a lookup-table activation: quantize input, apply the table, quantize output.""" + if not (table.quantize_output and table.enable_quantization): + name = getattr(table, 'name', None) or type(table).__name__ + raise PQuantAlkaidError(f'PQSoftmax table {name!r} must have an enabled output quantizer for Alkaid conversion.') + x = _replay_quantizer_if_enabled(table, 'input_quantizer', x, 'quantize_input') + out = x.apply(table_fn(table)) + return _replay_quantizer(table.output_quantizer, out) + + +def _replay_softmax(layer: Any, inputs: Any, table_fn: Any) -> Any: + """Replay PQSoftmax through its exp and inverse-sum lookup tables.""" + inputs = _replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + if layer.stable: + inputs = np.max(inputs, axis=layer.axes, keepdims=True) - inputs # type: ignore + exponents = _replay_table(layer.exp_table, inputs, table_fn) + sums = np.sum(exponents, axis=layer.axes, keepdims=True) + inverse_sums = _replay_table(layer.inv_table, sums, table_fn) + out = exponents * inverse_sums + return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + + +def _mark_plugin_loaded(framework: str) -> None: + """Record the pquant plugin as loaded in Alkaid's plugin loader, if that loader exists.""" + try: + from alkaid.converter import _plugin_loader + + _plugin_loader._LOADED.add(('pquant', framework)) + except Exception: + pass diff --git a/src/pquant/_alkaid_plugin/_alkaid_keras_plugin.py b/src/pquant/_alkaid_plugin/_alkaid_keras_plugin.py index 4e75957..716dc63 100644 --- a/src/pquant/_alkaid_plugin/_alkaid_keras_plugin.py +++ b/src/pquant/_alkaid_plugin/_alkaid_keras_plugin.py @@ -15,10 +15,14 @@ from pquant._alkaid_plugin._alkaid_common import ( PQuantAlkaidError, - replay_quantizer, - replay_quantizer_if_enabled, - to_bool, - to_numpy, + _assert_final_compression, + _final_bias, + _mark_plugin_loaded, + _replay_quantizer, + _replay_quantizer_if_enabled, + _replay_softmax, + _scale_by_relu_multiplier, + _to_numpy, ) from pquant.core.keras.activations import PQActivation from pquant.core.keras.layers import ( @@ -36,24 +40,31 @@ from pquant.core.keras.quantizer import Quantizer -def _assert_final_compression(layer) -> None: - if not to_bool(getattr(layer, 'final_compression_done', False)): - raise PQuantAlkaidError( - f'{layer.__class__.__name__} must have apply_final_compression() applied before Alkaid conversion.' - ) +def _final_kernel(layer) -> np.ndarray: + """The layer's final (compressed) kernel as numpy. Asserts compression was applied.""" + _assert_final_compression(layer) + return _to_numpy(layer._kernel) -def _weight(layer) -> np.ndarray: - _assert_final_compression(layer) - return to_numpy(layer._kernel) +def _table_fn(table): + """Numpy-callable for a PQActivation lookup table, evaluated in float32 like the keras runtime.""" + fn = table.activation_function + def apply_fn(v: np.ndarray) -> np.ndarray: + t = keras.ops.cast(keras.ops.convert_to_tensor(v), 'float32') + return np.asarray(keras.ops.convert_to_numpy(fn(t)), dtype=np.float64) + + return apply_fn -def _bias(layer) -> np.ndarray: - _assert_final_compression(layer) - bias = getattr(layer, '_bias', None) - if bias is None: - return np.array(0.0) - return to_numpy(bias) + +def _unpack_query_key_value(inputs): + if not isinstance(inputs, (list, tuple)): + return inputs, inputs, inputs + if len(inputs) == 3: + return inputs[0], inputs[1], inputs[2] + if len(inputs) == 2: + return inputs[0], inputs[1], inputs[1] + return inputs[0], inputs[0], inputs[0] class ReplayPQuantQuantizer(ReplayOperationBase): @@ -61,7 +72,7 @@ class ReplayPQuantQuantizer(ReplayOperationBase): handles = (Quantizer,) def call(self, x: FVArray) -> FVArray: - return replay_quantizer(self.op, x) + return _replay_quantizer(self.op, x) class ReplayPQuantDense(ReplayOperationBase): @@ -69,9 +80,9 @@ class ReplayPQuantDense(ReplayOperationBase): def call(self, inputs: FVArray) -> FVArray: layer = self.op - inputs = replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') - out = np.einsum('...c,cC->...C', inputs, _weight(layer)) + _bias(layer) - return replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + inputs = _replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + out = np.einsum('...c,cC->...C', inputs, _final_kernel(layer)) + _final_bias(layer) + return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') class ReplayPQuantConv(ReplayOperationBase): @@ -79,9 +90,9 @@ class ReplayPQuantConv(ReplayOperationBase): def call(self, inputs: FVArray) -> FVArray: layer = self.op - inputs = replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') - kernel = _weight(layer) - bias = _bias(layer) + inputs = _replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + kernel = _final_kernel(layer) + bias = _final_bias(layer) if isinstance(layer, (DepthwiseConv1D, DepthwiseConv2D)): ch_in, dm = kernel.shape[-2:] @@ -113,7 +124,7 @@ def call(self, inputs: FVArray) -> FVArray: out = out + bias if layer.data_format == 'channels_first': out = np.moveaxis(out, -1, 1) # type: ignore - return replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') class ReplayPQuantSeparableConv(ReplayOperationBase): @@ -131,14 +142,14 @@ class ReplayPQuantBatchNormalization(ReplayBatchNormalization): def fused_scale_offset(self) -> tuple[np.ndarray, np.ndarray]: layer = self.op _assert_final_compression(layer) - mean = to_numpy(keras.ops.cast(layer.moving_mean, layer.dtype)) - variance = to_numpy(keras.ops.cast(layer.moving_variance, layer.dtype)) + mean = _to_numpy(keras.ops.cast(layer.moving_mean, layer.dtype)) + variance = _to_numpy(keras.ops.cast(layer.moving_variance, layer.dtype)) if layer.scale: - gamma = to_numpy(keras.ops.cast(layer.gamma, layer.dtype)) + gamma = _to_numpy(keras.ops.cast(layer.gamma, layer.dtype)) else: gamma = np.ones_like(mean) if layer.center: - beta = to_numpy(keras.ops.cast(layer.beta, layer.dtype)) + beta = _to_numpy(keras.ops.cast(layer.beta, layer.dtype)) else: beta = np.zeros_like(mean) scale = gamma / np.sqrt(variance + layer.epsilon) @@ -147,7 +158,7 @@ def fused_scale_offset(self) -> tuple[np.ndarray, np.ndarray]: def call(self, inputs: FVArray, mask=None) -> FVArray: layer = self.op - inputs = replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + inputs = _replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') scale, offset = self.fused_scale_offset() shape = [1] * inputs.ndim axis = layer.axis if isinstance(layer.axis, (list, tuple)) else [layer.axis] @@ -168,9 +179,9 @@ class ReplayPQuantAvgPool(ReplayPool): def call(self, inputs: FVArray, mask: None = None) -> FVArray: layer = self.op - inputs = replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + inputs = _replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') out = super().call(inputs, mask=mask) - return replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') class ReplayPQuantActivation(ReplayOperationBase): @@ -179,57 +190,22 @@ class ReplayPQuantActivation(ReplayOperationBase): def call(self, inputs: FVArray) -> FVArray: layer = self.op - if ( - not bool(getattr(layer, 'use_hgq', False)) - and bool(getattr(layer, 'use_multiplier', False)) - and layer.activation_name == 'relu' - and hasattr(layer, 'multiplier') - ): - inputs = inputs * (2.0 ** np.rint(to_numpy(layer.multiplier))) - inputs = replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + inputs = _scale_by_relu_multiplier(layer, inputs) + inputs = _replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') if layer.activation_name not in keras_numpy_unary_map: raise PQuantAlkaidError(f'Unsupported PQuant activation for Alkaid conversion: {layer.activation_name!r}') out = keras_numpy_unary_map[layer.activation_name](inputs) - return replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') - - -def _table_fn(table): - """Numpy-callable for a PQActivation lookup table, evaluated in float32 like the keras runtime.""" - fn = table.activation_function - - def apply_fn(v: np.ndarray) -> np.ndarray: - t = keras.ops.cast(keras.ops.convert_to_tensor(v), 'float32') - return np.asarray(keras.ops.convert_to_numpy(fn(t)), dtype=np.float64) - - return apply_fn + return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') class ReplayPQuantSoftmax(ReplayOperationBase): __activation_handled__ = True handles = (PQSoftmax,) - @staticmethod - def _replay_table(table, x: FVArray) -> FVArray: - if not (table.quantize_output and table.enable_quantization): - raise PQuantAlkaidError( - f'PQSoftmax table {table.name!r} must have an enabled output quantizer for Alkaid conversion.' - ) - x = replay_quantizer_if_enabled(table, 'input_quantizer', x, 'quantize_input') - out = x.apply(_table_fn(table)) - return replay_quantizer(table.output_quantizer, out) - def call(self, inputs: FVArray, mask=None) -> FVArray: - layer = self.op if mask is not None: raise PQuantAlkaidError('PQSoftmax masks are not supported in Alkaid conversion.') - inputs = replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') - if layer.stable: - inputs = np.max(inputs, axis=layer.axes, keepdims=True) - inputs # type: ignore - exp_inp = self._replay_table(layer.exp_table, inputs) - sums = np.sum(exp_inp, axis=layer.axes, keepdims=True) - divisor = self._replay_table(layer.inv_table, sums) - out = exp_inp * divisor - return replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + return _replay_softmax(self.op, inputs, _table_fn) class ReplayPQuantMultiheadAttention(ReplayOperationBase): @@ -241,17 +217,7 @@ def call(self, inputs, key_padding_mask=None, attn_mask=None, need_weights=True) if key_padding_mask is not None or attn_mask is not None: raise PQuantAlkaidError('Attention masks are not supported in Alkaid conversion.') - if isinstance(inputs, (list, tuple)): - if len(inputs) == 3: - query, key, value = inputs - elif len(inputs) == 2: - query, key = inputs - value = key - else: - query = key = value = inputs[0] - else: - query = key = value = inputs - + query, key, value = _unpack_query_key_value(inputs) batch_size, query_len = query.shape[0], query.shape[1] key_len = key.shape[1] num_heads, head_dim = layer.num_heads, layer.head_dim @@ -286,10 +252,4 @@ def call(self, inputs, key_padding_mask=None, attn_mask=None, need_weights=True) def register() -> None: """Entry point for Alkaid's ``alkaid_keras`` second-level plugin group.""" - try: - from alkaid.converter import _plugin_loader - - _plugin_loader._LOADED.add(('pquant', 'keras')) - except Exception: - pass - return None + _mark_plugin_loaded('keras') diff --git a/src/pquant/_alkaid_plugin/_alkaid_torch_plugin.py b/src/pquant/_alkaid_plugin/_alkaid_torch_plugin.py index 22428e6..4f24183 100644 --- a/src/pquant/_alkaid_plugin/_alkaid_torch_plugin.py +++ b/src/pquant/_alkaid_plugin/_alkaid_torch_plugin.py @@ -7,172 +7,157 @@ import numpy as np import torch - -try: - from torch.fx._symbolic_trace import is_fx_symbolic_tracing -except ImportError: # torch < 2.8 exposes it as is_fx_tracing - from torch.fx._symbolic_trace import is_fx_tracing as is_fx_symbolic_tracing -from alkaid.converter.builtin.torch.layers.functional import _functional_map +from alkaid.converter.builtin.torch.layers.direct import torch_numpy_unary_map +from alkaid.converter.builtin.torch.layers.functional import ( + _functional_map, + conv_nd_replay, + replay_avg_pool, +) from alkaid.converter.builtin.torch.layers.methods import _method_map -from alkaid.converter.builtin.torch.layers.modules import ReplayModuleBase +from alkaid.converter.builtin.torch.layers.modules import ( + ReplayBatchNorm, + ReplayModuleBase, +) from alkaid.trace import FVArray from pquant._alkaid_plugin._alkaid_common import ( PQuantAlkaidError, - replay_quantizer, - replay_quantizer_if_enabled, + _assert_final_compression, + _final_bias, + _mark_plugin_loaded, + _replay_quantizer, + _replay_quantizer_if_enabled, + _replay_softmax, + _scale_by_relu_multiplier, + _to_numpy, ) -from pquant.core.torch.activations import PQActivation +from pquant.core.torch.activations import PQActivation, PQSoftmax from pquant.core.torch.layers import ( + PQAvgPool1d, + PQAvgPool2d, PQBatchNorm1d, PQBatchNorm2d, PQConv1d, PQConv2d, PQDense, - PQSoftmax, - PQWeightBiasBase, ) from pquant.core.torch.quantizer import Quantizer -def _contains_fx_proxy(value: Any) -> bool: - from torch.fx.proxy import Proxy +def _final_weight(layer: torch.nn.Module) -> np.ndarray: + """The layer's final (compressed) weight as numpy. Asserts compression was applied.""" + _assert_final_compression(layer) + return _to_numpy(layer._weight) - if isinstance(value, Proxy): - return True - if isinstance(value, (tuple, list)): - return any(_contains_fx_proxy(v) for v in value) - if isinstance(value, dict): - return any(_contains_fx_proxy(v) for v in value.values()) - return False +def _activation_numpy_fn(layer: PQActivation): + """The numpy elementwise function for a named PQActivation (relu/tanh/gelu/...).""" + name = layer.activation_name + fn = torch_numpy_unary_map.get(name) or torch_numpy_unary_map.get(name.replace('_', '')) + if fn is not None: + return fn + if name == 'leaky_relu': + slope = float(getattr(layer.activation_function, 'negative_slope', 0.1015625)) + return lambda x: np.where(x < 0, x * slope, x) # type: ignore + raise PQuantAlkaidError(f'Unsupported PQuant activation for Alkaid conversion: {name!r}') -def _patch_once(cls: type, name: str, wrapper_factory) -> None: - marker = f'__alkaid_pquant_patched_{name}__' - if getattr(cls, marker, False): - return - original = getattr(cls, name) - setattr(cls, f'__alkaid_pquant_original_{name}__', original) - setattr(cls, name, wrapper_factory(original)) - setattr(cls, marker, True) +def _table_fn(table): + """Numpy-callable for a PQActivation lookup table, evaluated in float32 like the torch runtime.""" + fn = table.activation_function -def _module_parameter(module: torch.nn.Module, name: str) -> Any: - if name in module._parameters: - return module._parameters[name] - return getattr(module, name) + def apply_fn(v: np.ndarray) -> np.ndarray: + with torch.no_grad(): + t = torch.as_tensor(v, dtype=torch.float32, device='cpu') + return fn(t).detach().cpu().numpy().astype(np.float64) + return apply_fn -def _module_bool(module: torch.nn.Module, name: str, default: bool = False) -> bool: - value = module._parameters.get(name, getattr(module, name, default)) - if isinstance(value, torch.Tensor): - return bool(value.detach().cpu().item()) - return bool(value) +class ReplayPQuantQuantizer(ReplayModuleBase): + handles = (Quantizer,) -def _assert_final_compression(module: torch.nn.Module) -> None: - if not _module_bool(module, 'final_compression_done'): - raise PQuantAlkaidError( - f'{type(module).__name__} must have apply_final_compression() applied before Alkaid conversion.' - ) + def call(self, input: FVArray) -> FVArray: + return _replay_quantizer(self.module, input) -def _patch_weight_bias_properties() -> None: - for cls in (PQDense, PQConv1d, PQConv2d, PQBatchNorm1d, PQBatchNorm2d): - marker = '__alkaid_pquant_patched_weight_bias__' - if getattr(cls, marker, False): - continue - original_weight = cls.weight.fget - original_bias = cls.bias.fget - - def weight(self, _original_weight=original_weight): - if not is_fx_symbolic_tracing(): - return _original_weight(self) - _assert_final_compression(self) - return _module_parameter(self, '_weight') - - def bias(self, _original_bias=original_bias): - if not is_fx_symbolic_tracing(): - return _original_bias(self) - _assert_final_compression(self) - return _module_parameter(self, '_bias') - - cls.weight = property(weight) - cls.bias = property(bias) - setattr(cls, marker, True) - - -def _patch_lazy_build_assertions() -> None: - def wrap_pre_forward(original): - @wraps(original) - def wrapped(self, x): - if not _contains_fx_proxy(x): - return original(self, x) - if self.quantize_input: - x = self.quantize(x, self.input_quantizer) - return x - - return wrapped - - _patch_once(PQWeightBiasBase, 'pre_forward', wrap_pre_forward) - - def wrap_pre_activation(original): - @wraps(original) - def wrapped(self, x): - if not _contains_fx_proxy(x): - return original(self, x) - if not self.use_hgq and self.use_multiplier and self.activation_name == 'relu' and hasattr(self, 'multiplier'): - multiplier = _module_parameter(self, 'multiplier') - x = x * (2.0 ** torch.round(multiplier.detach()).item()) - if self.quantize_input and self.enable_quantization: - x = self.input_quantizer(x) - return x - - return wrapped - - _patch_once(PQActivation, 'pre_activation', wrap_pre_activation) - - def wrap_bn_forward(original): - @wraps(original) - def wrapped(self, input): - if not _contains_fx_proxy(input): - return original(self, input) - if self.quantize_input and self.enable_quantization: - input = self.input_quantizer(input) - return torch.nn.functional.batch_norm( - input, - self.running_mean, - self.running_var, - self.weight, - self.bias, - False, - self.momentum, - self.eps, - ) - - return wrapped - - _patch_once(PQBatchNorm1d, 'forward', wrap_bn_forward) - _patch_once(PQBatchNorm2d, 'forward', wrap_bn_forward) +class ReplayPQuantDense(ReplayModuleBase): + handles = (PQDense,) + def call(self, input: FVArray) -> FVArray: + layer = self.module + input = _replay_quantizer_if_enabled(layer, 'input_quantizer', input, 'quantize_input') + out = input @ _final_weight(layer).T + bias = _final_bias(layer) + if bias.shape != (): + out = out + bias + return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') -class ReplayPQuantQuantizer(ReplayModuleBase): - handles = (Quantizer,) + +class ReplayPQuantConv(ReplayModuleBase): + handles = (PQConv1d, PQConv2d) def call(self, input: FVArray) -> FVArray: - return replay_quantizer(self.module, input) + layer = self.module + input = _replay_quantizer_if_enabled(layer, 'input_quantizer', input, 'quantize_input') + out = conv_nd_replay( + input, + _final_weight(layer), + _final_bias(layer), + stride=layer.stride, + padding=layer.padding, + dilation=layer.dilation, + groups=layer.groups, + ) + return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') -def _table_fn(table): - """Numpy-callable for a PQActivation lookup table, evaluated in float32 like the torch runtime.""" - fn = table.activation_function +class ReplayPQuantBatchNorm(ReplayBatchNorm): + handles = (PQBatchNorm1d, PQBatchNorm2d) - def apply_fn(v: np.ndarray) -> np.ndarray: - with torch.no_grad(): - t = torch.as_tensor(v, dtype=torch.float32, device='cpu') - return fn(t).detach().cpu().numpy().astype(np.float64) + def fused_scale_offset(self) -> tuple[np.ndarray, np.ndarray]: + layer = self.module + _assert_final_compression(layer) + mean = _to_numpy(layer.running_mean) + variance = _to_numpy(layer.running_var) + gamma = _to_numpy(layer._weight) if layer._weight is not None else np.ones_like(mean) + beta = _to_numpy(layer._bias) if layer._bias is not None else np.zeros_like(mean) + scale = gamma / np.sqrt(variance + layer.eps) + offset = beta - mean * scale + return scale, offset + + def call(self, input: FVArray) -> FVArray: + layer = self.module + input = _replay_quantizer_if_enabled(layer, 'input_quantizer', input, 'quantize_input') + return super().call(input) - return apply_fn + +class ReplayPQuantAvgPool(ReplayModuleBase): + handles = (PQAvgPool1d, PQAvgPool2d) + + def call(self, input: FVArray) -> FVArray: + layer = self.module + input = _replay_quantizer_if_enabled(layer, 'input_quantizer', input, 'quantize_input') + out = replay_avg_pool( + input, + layer.kernel_size, + layer.stride, + layer.padding, + layer.ceil_mode, + layer.count_include_pad, + ) + return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + + +class ReplayPQuantActivation(ReplayModuleBase): + handles = (PQActivation,) + + def call(self, input: FVArray) -> FVArray: + layer = self.module + input = _scale_by_relu_multiplier(layer, input) + input = _replay_quantizer_if_enabled(layer, 'input_quantizer', input, 'quantize_input') + out = _activation_numpy_fn(layer)(input) + return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') class ReplayPQuantSoftmax(ReplayModuleBase): @@ -180,33 +165,17 @@ class ReplayPQuantSoftmax(ReplayModuleBase): handles = (PQSoftmax,) - @staticmethod - def _replay_table(table, x: FVArray) -> FVArray: - if not (table.quantize_output and table.enable_quantization): - raise PQuantAlkaidError( - f'PQSoftmax table {type(table).__name__} must have an enabled output quantizer for Alkaid conversion.' - ) - x = replay_quantizer_if_enabled(table, 'input_quantizer', x, 'quantize_input') - out = x.apply(_table_fn(table)) - return replay_quantizer(table.output_quantizer, out) - def call(self, inputs: FVArray, mask=None) -> FVArray: module = self.module if mask is not None: raise PQuantAlkaidError('PQSoftmax masks are not supported in Alkaid conversion.') if not module.built: raise PQuantAlkaidError('PQSoftmax must be built (one real forward) before Alkaid conversion.') - inputs = replay_quantizer_if_enabled(module, 'input_quantizer', inputs, 'quantize_input') - if module.stable: - inputs = np.max(inputs, axis=module.axes, keepdims=True) - inputs # type: ignore - exp_inp = self._replay_table(module.exp_table, inputs) - sums = np.sum(exp_inp, axis=module.axes, keepdims=True) - divisor = self._replay_table(module.inv_table, sums) - out = exp_inp * divisor - return replay_quantizer_if_enabled(module, 'output_quantizer', out, 'quantize_output') + return _replay_softmax(module, inputs, _table_fn) def _patch_root_quantizer_trace() -> None: + """Let a bare Quantizer be traced as the model root, which Alkaid's fx tracer cannot handle.""" import alkaid.converter.builtin.torch.main as torch_main tracer_cls = torch_main.TorchALIRTracer @@ -235,7 +204,7 @@ def _replay_getattr(obj: Any, name: str, *default: Any) -> Any: return getattr(obj, name) -def _tensor(data: Any, *args: Any, **kwargs: Any) -> Any: +def _replay_tensor(data: Any, *args: Any, **kwargs: Any) -> Any: if isinstance(data, FVArray): return data if isinstance(data, torch.Tensor): @@ -249,23 +218,23 @@ def _normalize_shape(args: tuple[Any, ...]) -> tuple[int, ...]: return tuple(int(v) for v in args) -def _zeros(*size: Any, **kwargs: Any) -> np.ndarray: +def _replay_zeros(*size: Any, **kwargs: Any) -> np.ndarray: return np.zeros(_normalize_shape(size), dtype=np.float32) -def _ones(*size: Any, **kwargs: Any) -> np.ndarray: +def _replay_ones(*size: Any, **kwargs: Any) -> np.ndarray: return np.ones(_normalize_shape(size), dtype=np.float32) -def _full(size: Any, fill_value: Any, **kwargs: Any) -> np.ndarray: +def _replay_full(size: Any, fill_value: Any, **kwargs: Any) -> np.ndarray: return np.full(_normalize_shape((size,)), fill_value, dtype=np.float32) -def _zeros_like(x: Any, **kwargs: Any) -> np.ndarray: +def _replay_zeros_like(x: Any, **kwargs: Any) -> np.ndarray: return np.zeros(tuple(x.shape), dtype=np.float32) -def _ones_like(x: Any, **kwargs: Any) -> np.ndarray: +def _replay_ones_like(x: Any, **kwargs: Any) -> np.ndarray: return np.ones(tuple(x.shape), dtype=np.float32) @@ -273,25 +242,18 @@ def _register_functional_helpers() -> None: _functional_map.setdefault(operator.pow, lambda a, b: a**b) _functional_map.setdefault(torch.pow, lambda a, b: a**b) _functional_map.setdefault(builtins.getattr, _replay_getattr) - _functional_map.setdefault(torch.tensor, _tensor) - _functional_map.setdefault(torch.as_tensor, _tensor) - _functional_map.setdefault(torch.zeros, _zeros) - _functional_map.setdefault(torch.ones, _ones) - _functional_map.setdefault(torch.full, _full) - _functional_map.setdefault(torch.zeros_like, _zeros_like) - _functional_map.setdefault(torch.ones_like, _ones_like) + _functional_map.setdefault(torch.tensor, _replay_tensor) + _functional_map.setdefault(torch.as_tensor, _replay_tensor) + _functional_map.setdefault(torch.zeros, _replay_zeros) + _functional_map.setdefault(torch.ones, _replay_ones) + _functional_map.setdefault(torch.full, _replay_full) + _functional_map.setdefault(torch.zeros_like, _replay_zeros_like) + _functional_map.setdefault(torch.ones_like, _replay_ones_like) _method_map.setdefault('pow', lambda receiver, exponent, **_kwargs: receiver**exponent) def register() -> None: """Entry point for Alkaid's ``alkaid_torch`` second-level plugin group.""" - _patch_lazy_build_assertions() - _patch_weight_bias_properties() _patch_root_quantizer_trace() _register_functional_helpers() - try: - from alkaid.converter import _plugin_loader - - _plugin_loader._LOADED.add(('pquant', 'torch')) - except Exception: - pass + _mark_plugin_loaded('torch') diff --git a/src/pquant/core/keras/layers.py b/src/pquant/core/keras/layers.py index 4a058c0..5615d80 100644 --- a/src/pquant/core/keras/layers.py +++ b/src/pquant/core/keras/layers.py @@ -1435,12 +1435,12 @@ def build(self, input_shape): self.input_shape = (1,) + tuple(input_shape[1:]) def apply_final_compression(self): - self.final_compression_done = True if self.enable_quantization and self.quantize_parameters: if self.gamma is not None: self.gamma.assign(self.weight_quantizer(self.gamma)) if self.beta is not None: self.beta.assign(self.bias_quantizer(self.beta)) + self.final_compression_done = True def ebops(self): bw_inp = self.input_quantizer.get_total_bits(self.input_shape) diff --git a/src/pquant/core/torch/layers.py b/src/pquant/core/torch/layers.py index 7964172..a6a969e 100644 --- a/src/pquant/core/torch/layers.py +++ b/src/pquant/core/torch/layers.py @@ -976,9 +976,9 @@ def check_is_built(self, input_shape): self.input_shape = (1,) + input_shape[1:] def apply_final_compression(self): - self.final_compression_done = True self._weight.data = self.weight self._bias.data = self.bias + self.final_compression_done = True def get_input_quantization_bits(self): return self.input_quantizer.get_quantization_bits() @@ -1150,9 +1150,9 @@ def check_is_built(self, input_shape): self.input_shape = (1,) + input_shape[1:] def apply_final_compression(self): - self.final_compression_done = True self._weight.data = self.weight self._bias.data = self.bias + self.final_compression_done = True def get_input_quantization_bits(self): return self.input_quantizer.get_quantization_bits() @@ -1352,11 +1352,11 @@ def check_is_built(self, input_shape): self.input_shape = (1,) + tuple(input_shape[1:]) def apply_final_compression(self): - self.final_compression_done = True if self._weight is not None: self._weight.data = self.weight if self._bias is not None: self._bias.data = self.bias + self.final_compression_done = True def get_input_quantization_bits(self): return self.input_quantizer.get_quantization_bits() diff --git a/tests/test_keras_alkaid_conversion.py b/tests/test_keras_alkaid_conversion.py index b96c185..04a866b 100644 --- a/tests/test_keras_alkaid_conversion.py +++ b/tests/test_keras_alkaid_conversion.py @@ -41,12 +41,12 @@ @pytest.fixture(autouse=True) -def _channels_last(): - # Override conftest's default (channels_first); see module docstring. +def channels_last(): + # We assume Keras models are channels_last keras.backend.set_image_data_format("channels_last") -def _build_model(config): +def build_model(config): img_in = keras.Input(shape=IMG_SHAPE, name="img") a = PQConv2d(config, OUT_FEATURES, KERNEL_SIZE, padding="same")(img_in) a = PQActivation(config, activation="relu", quantize_input=True, quantize_output=True)(a) @@ -63,7 +63,7 @@ def _build_model(config): return keras.Model([img_in, seq_in], x) -def _rtl_predict(comb, path, data): +def rtl_predict(comb, path, data): """Write the RTL project, compile the simulation emulator, and run bit-accurate inference.""" rtl_model = RTLModel(comb, str(path), "model", flavor="verilog", latency_cutoff=5, clock_period=5.0, print_latency=False) rtl_model.write() @@ -75,7 +75,7 @@ def _rtl_predict(comb, path, data): return rtl_model.predict(data) -def _random_prune(layer, fraction, rng): +def random_prune(layer, fraction, rng): """Zero exactly ``fraction`` of the layer's weights via its pruning mask.""" mask = layer.pruning_layer.mask numel = int(np.prod(mask.shape)) @@ -86,36 +86,20 @@ def _random_prune(layer, fraction, rng): return n_zero / numel -def _build_pruned_compressed_model(config, rng): - """Build the model, build it (one forward), prune 90%, and apply final compression.""" - model = _build_model(config) - +def build_pruned_compressed_model(config): + model = build_model(config) img = np.zeros((1,) + IMG_SHAPE, dtype="float32") seq = np.zeros((1,) + SEQ_SHAPE, dtype="float32") - - # Call once to build the quantizers and pruning masks. model([img, seq]) - - pq_layers = [layer for layer in model.layers if isinstance(layer, (PQConv2d, PQConv1d, PQDense))] - - for layer in pq_layers: - layer._kernel.assign(rng.standard_normal(layer._kernel.shape).astype("float32")) - expected_sparsity = {layer.name: _random_prune(layer, PRUNE_FRACTION, rng) for layer in pq_layers} - apply_final_compression(model) - return model, pq_layers, expected_sparsity + return model def test_alkaid_conversion_pruned_quantized_model(): config = pdp_config() config.quantization_parameters.enable_quantization = True - - rng = np.random.default_rng(0) - model, pq_layers, expected_sparsity = _build_pruned_compressed_model(config, rng) - assert {type(layer).__name__ for layer in pq_layers} == {"PQConv2d", "PQConv1d", "PQDense"} - + model = build_pruned_compressed_model(config) inp, out = trace_model(model, inputs_kif=INPUT_KIF) - assert out.shape == (OUT_FEATURES,) assert inp.shape == (int(np.prod(IMG_SHAPE)) + int(np.prod(SEQ_SHAPE)),) @@ -124,26 +108,24 @@ def test_alkaid_rtl_matches_model(tmp_path): config = pdp_config() config.quantization_parameters.enable_quantization = True - rng = np.random.default_rng(0) - model, _, _ = _build_pruned_compressed_model(config, rng) + model = build_pruned_compressed_model(config) inp_fv, out_fv = trace_model(model, inputs_kif=INPUT_KIF) comb = trace(inp_fv, out_fv, optimize=True) n_samples = 16 + rng = np.random.default_rng(0) img = rng.integers(0, 16, size=(n_samples,) + IMG_SHAPE).astype("float32") / 16.0 seq = rng.integers(0, 16, size=(n_samples,) + SEQ_SHAPE).astype("float32") / 16.0 reference = np.asarray(model([img, seq]), dtype=np.float64) # (n_samples, OUT_FEATURES) - emulated = _rtl_predict(comb, tmp_path, [img, seq]) + emulated = rtl_predict(comb, tmp_path, [img, seq]) assert (tmp_path / "src" / "model.v").exists() assert np.any(reference != 0) # the comparison is non-trivial np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) -# --- Coverage of every PQ layer the keras Alkaid plugin handles --------------- - ALL_C = 4 ALL_H = ALL_W = 8 ALL_LIN = (ALL_H // 2) * (ALL_W // 2) * 2 @@ -164,7 +146,7 @@ def test_alkaid_rtl_matches_model(tmp_path): } -def _build_all_layers_model(config): +def build_all_layers_model(config): """Model exercising every PQ layer type the keras Alkaid plugin handles.""" img_in = keras.Input(shape=ALL_IMG_SHAPE, name="img") a = PQConv2d(config, ALL_C, KERNEL_SIZE, padding="same")(img_in) @@ -187,31 +169,13 @@ def _build_all_layers_model(config): return keras.Model([img_in, seq_in], x) -def _all_prunable_layers(model): - """Every layer with a pruning mask, descending into PQSeparableConv2d's sub-convs.""" - found = [] - - def visit(layer): - if getattr(layer, "pruning_layer", None) is not None: - found.append(layer) - for name in ("depthwise_conv", "pointwise_conv"): - sub = getattr(layer, name, None) - if sub is not None: - visit(sub) - - for layer in model.layers: - visit(layer) - return found - - def test_alkaid_conversion_all_layer_types(tmp_path): config = pdp_config() config.quantization_parameters.enable_quantization = True - model = _build_all_layers_model(config) + model = build_all_layers_model(config) rng = np.random.default_rng(0) - # Build with random input so batchnorm running stats are sane. model( [ rng.standard_normal((4,) + ALL_IMG_SHAPE).astype("float32"), @@ -219,34 +183,29 @@ def test_alkaid_conversion_all_layer_types(tmp_path): ] ) - assert ALL_KERAS_LAYER_TYPES <= {type(layer).__name__ for layer in model.layers} - - for layer in _all_prunable_layers(model): + for layer in model._flatten_layers(): + if not hasattr(layer, "pruning_layer"): + continue layer._kernel.assign(rng.standard_normal(layer._kernel.shape).astype("float32")) - _random_prune(layer, PRUNE_FRACTION, rng) + random_prune(layer, PRUNE_FRACTION, rng) apply_final_compression(model) inp_fv, out_fv = trace_model(model, inputs_kif=INPUT_KIF) comb = trace(inp_fv, out_fv, optimize=True) - assert out_fv.shape == (OUT_FEATURES,) n_samples = 16 img = rng.integers(0, 16, size=(n_samples,) + ALL_IMG_SHAPE).astype("float32") / 16.0 seq = rng.integers(0, 16, size=(n_samples,) + ALL_SEQ_SHAPE).astype("float32") / 16.0 reference = np.asarray(model([img, seq]), dtype=np.float64) - emulated = _rtl_predict(comb, tmp_path, [img, seq]) + emulated = rtl_predict(comb, tmp_path, [img, seq]) assert (tmp_path / "src" / "model.v").exists() assert np.any(reference != 0) np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) -# --- Per-layer conversion: a model that is a single layer --------------------- - - -def _data_quantizer(config): - """A data Quantizer built from the config's default data settings.""" +def make_data_quantizer(config): qp = config.quantization_parameters return Quantizer( k=qp.default_data_keep_negatives, @@ -262,61 +221,59 @@ def _data_quantizer(config): ) -def _single_layer_model(input_shape, layer, tail=None): - """A keras model that is one PQ layer, optionally followed by a Quantizer.""" +def create_single_layer_model(input_shape, layer, out_quantizer=False): + """A keras model that is one PQ layer, optionally followed by a data Quantizer.""" inp = keras.Input(shape=input_shape) x = layer(inp) - if tail is not None: - x = tail(x) + if out_quantizer: + x = make_data_quantizer(layer.config)(x) return keras.Model(inp, x) -# id -> lambda(config) -> (input shape without batch, single-layer model). -# Layers with quantize_output set it; batchnorm (which has none) gets a trailing Quantizer. -_SINGLE_LAYER_CASES = { +SINGLE_LAYER_CASES = { "conv2d": lambda c: ( (4, 4, 2), - _single_layer_model((4, 4, 2), PQConv2d(c, 3, KERNEL_SIZE, padding="same", quantize_output=True)), + create_single_layer_model((4, 4, 2), PQConv2d(c, 3, KERNEL_SIZE, padding="same", quantize_output=True)), ), "conv1d": lambda c: ( (8, 2), - _single_layer_model((8, 2), PQConv1d(c, 3, KERNEL_SIZE, padding="same", quantize_output=True)), + create_single_layer_model((8, 2), PQConv1d(c, 3, KERNEL_SIZE, padding="same", quantize_output=True)), ), - "dense": lambda c: ((6,), _single_layer_model((6,), PQDense(c, units=OUT_FEATURES, quantize_output=True))), + "dense": lambda c: ((6,), create_single_layer_model((6,), PQDense(c, units=OUT_FEATURES, quantize_output=True))), "depthwise2d": lambda c: ( (4, 4, 3), - _single_layer_model((4, 4, 3), PQDepthwiseConv2d(c, KERNEL_SIZE, padding="same", quantize_output=True)), + create_single_layer_model((4, 4, 3), PQDepthwiseConv2d(c, KERNEL_SIZE, padding="same", quantize_output=True)), ), "separable2d": lambda c: ( (4, 4, 2), - _single_layer_model((4, 4, 2), PQSeparableConv2d(c, 3, KERNEL_SIZE, padding="same", quantize_output=True)), + create_single_layer_model((4, 4, 2), PQSeparableConv2d(c, 3, KERNEL_SIZE, padding="same", quantize_output=True)), ), - "batchnorm": lambda c: ((6,), _single_layer_model((6,), PQBatchNormalization(c, axis=-1), _data_quantizer(c))), + "batchnorm": lambda c: ((6,), create_single_layer_model((6,), PQBatchNormalization(c, axis=-1), out_quantizer=True)), "avgpool2d": lambda c: ( (4, 4, 3), - _single_layer_model((4, 4, 3), PQAvgPool2d(c, pool_size=2, strides=2, quantize_output=True)), + create_single_layer_model((4, 4, 3), PQAvgPool2d(c, pool_size=2, strides=2, quantize_output=True)), ), "avgpool1d": lambda c: ( (8, 3), - _single_layer_model((8, 3), PQAvgPool1d(c, pool_size=2, strides=2, quantize_output=True)), + create_single_layer_model((8, 3), PQAvgPool1d(c, pool_size=2, strides=2, quantize_output=True)), ), "activation": lambda c: ( (6,), - _single_layer_model((6,), PQActivation(c, activation="relu", quantize_input=True, quantize_output=True)), + create_single_layer_model((6,), PQActivation(c, activation="relu", quantize_input=True, quantize_output=True)), ), - "quantizer": lambda c: ((6,), _single_layer_model((6,), _data_quantizer(c))), - "softmax": lambda c: ((6,), _single_layer_model((6,), PQSoftmax(c, axis=-1))), + "quantizer": lambda c: ((6,), create_single_layer_model((6,), make_data_quantizer(c))), + "softmax": lambda c: ((6,), create_single_layer_model((6,), PQSoftmax(c, axis=-1))), } -@pytest.mark.parametrize("case_id", list(_SINGLE_LAYER_CASES)) +@pytest.mark.parametrize("case_id", list(SINGLE_LAYER_CASES)) def test_alkaid_single_layer(case_id, tmp_path): config = pdp_config() config.quantization_parameters.enable_quantization = True - input_shape, model = _SINGLE_LAYER_CASES[case_id](config) + input_shape, model = SINGLE_LAYER_CASES[case_id](config) rng = np.random.default_rng(0) - model(rng.standard_normal((4,) + input_shape).astype("float32")) # build + model(rng.standard_normal((4,) + input_shape).astype("float32")) apply_final_compression(model) inp_fv, out_fv = trace_model(model, inputs_kif=INPUT_KIF) @@ -325,21 +282,18 @@ def test_alkaid_single_layer(case_id, tmp_path): n_samples = 16 x = rng.integers(0, 16, size=(n_samples,) + input_shape).astype("float32") / 16.0 reference = np.asarray(model(x), dtype=np.float64).reshape(n_samples, -1) - emulated = _rtl_predict(comb, tmp_path, x) + emulated = rtl_predict(comb, tmp_path, x) assert np.any(reference != 0) np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) -# --- Multi-head attention ------------------------------------------------------ - MHA_SEQ_LEN = 4 MHA_EMBED_DIM = 4 MHA_NUM_HEADS = 2 -def _build_mha_model(config, rng): - """Self-attention PQMultiheadAttention model with every data quantizer enabled.""" +def build_mha_model(config, rng): inp = keras.Input(shape=(MHA_SEQ_LEN, MHA_EMBED_DIM)) out, _ = PQMultiheadAttention( config, @@ -348,7 +302,7 @@ def _build_mha_model(config, rng): quantize_output=True, )(inp) model = keras.Model(inp, out) - model(rng.standard_normal((4, MHA_SEQ_LEN, MHA_EMBED_DIM)).astype("float32")) # build + model(rng.standard_normal((4, MHA_SEQ_LEN, MHA_EMBED_DIM)).astype("float32")) apply_final_compression(model) return model @@ -358,7 +312,7 @@ def test_alkaid_multihead_attention(tmp_path): config.quantization_parameters.enable_quantization = True rng = np.random.default_rng(0) - model = _build_mha_model(config, rng) + model = build_mha_model(config, rng) inp_fv, out_fv = trace_model(model, inputs_kif=INPUT_KIF) comb = trace(inp_fv, out_fv, optimize=True) @@ -367,7 +321,7 @@ def test_alkaid_multihead_attention(tmp_path): n_samples = 16 x = rng.integers(0, 16, size=(n_samples, MHA_SEQ_LEN, MHA_EMBED_DIM)).astype("float32") / 16.0 reference = np.asarray(model(x), dtype=np.float64).reshape(n_samples, -1) - emulated = _rtl_predict(comb, tmp_path, x) + emulated = rtl_predict(comb, tmp_path, x) assert (tmp_path / "src" / "model.v").exists() assert np.any(reference != 0) diff --git a/tests/test_torch_alkaid_conversion.py b/tests/test_torch_alkaid_conversion.py index dfb0370..dfeafd1 100644 --- a/tests/test_torch_alkaid_conversion.py +++ b/tests/test_torch_alkaid_conversion.py @@ -40,12 +40,6 @@ class TwoBranchNet(nn.Module): - """conv2d branch + conv1d branch merged (matched flatten lengths) -> dense head. - - A conv after a reshape/flatten cannot be traced by Alkaid (its reshape folds - the batch axis), and its ``Concatenate`` merge is unreliable, so the two convs - live on separate branches that are summed before the dense head. - """ def __init__(self, config): super().__init__() @@ -67,8 +61,7 @@ def forward(self, img, seq): return self.act(self.dense(x)) -def _random_prune(layer, fraction, rng): - """Zero exactly ``fraction`` of the layer's weights via its pruning mask.""" +def random_prune(layer, fraction, rng): mask = layer.pruning_layer.mask numel = int(np.prod(tuple(mask.shape))) n_zero = int(round(fraction * numel)) @@ -78,14 +71,13 @@ def _random_prune(layer, fraction, rng): return n_zero / numel -def _fixed_point_input(shape, kif=INPUT_KIF): +def fixed_point_input(shape, kif=INPUT_KIF): """Bounded fixed-point symbolic input so the SAT input quantizer can be replayed.""" k, i, f = (np.full(shape, v, dtype=np.int8) for v in kif) return FVArray.from_kif(k, i, f, HWCONF, 0, None) -def _rtl_predict(comb, path, data): - """Write the RTL project, compile the simulation emulator, and run bit-accurate inference.""" +def rtl_predict(comb, path, data): rtl_model = RTLModel(comb, str(path), "model", flavor="verilog", latency_cutoff=5, clock_period=5.0, print_latency=False) rtl_model.write() rtl_model.compile() @@ -96,36 +88,25 @@ def _rtl_predict(comb, path, data): return rtl_model.predict(data) -def _build_pruned_compressed_model(config, rng): - """Build the model, build it (one forward), prune 90%, apply final compression, eval.""" +def build_pruned_compressed_model(config): model = TwoBranchNet(config) device = next(model.parameters()).device img = torch.zeros(1, IN_FEATURES, H, W, device=device) seq = torch.zeros(1, IN_FEATURES, SEQ_LEN, device=device) - with torch.no_grad(): - model(img, seq) # build quantizers + pruning masks - - pq_layers = [m for m in model.modules() if isinstance(m, (PQConv2d, PQConv1d, PQDense))] - for layer in pq_layers: - layer._weight.copy_( - torch.tensor(rng.standard_normal(tuple(layer._weight.shape)), dtype=layer._weight.dtype, device=device) - ) - expected_sparsity = {id(layer): _random_prune(layer, PRUNE_FRACTION, rng) for layer in pq_layers} - + model(img, seq) apply_final_compression(model) model.eval() - return model, pq_layers, expected_sparsity, device + return model, device def test_alkaid_conversion_pruned_quantized_model(): config = pdp_config() config.quantization_parameters.enable_quantization = True - rng = np.random.default_rng(0) - model, _, _, _ = _build_pruned_compressed_model(config, rng) + model, _ = build_pruned_compressed_model(config) - inputs = (_fixed_point_input((1, IN_FEATURES, H, W)), _fixed_point_input((1, IN_FEATURES, SEQ_LEN))) + inputs = (fixed_point_input((1, IN_FEATURES, H, W)), fixed_point_input((1, IN_FEATURES, SEQ_LEN))) inp, out = trace_model(model, hwconf=HWCONF, inputs=inputs, framework="torch") assert out.shape == (OUT_FEATURES,) @@ -138,9 +119,9 @@ def test_alkaid_rtl_matches_model(tmp_path): config.quantization_parameters.enable_quantization = True rng = np.random.default_rng(0) - model, _, _, device = _build_pruned_compressed_model(config, rng) + model, device = build_pruned_compressed_model(config) - inputs = (_fixed_point_input((1, IN_FEATURES, H, W)), _fixed_point_input((1, IN_FEATURES, SEQ_LEN))) + inputs = (fixed_point_input((1, IN_FEATURES, H, W)), fixed_point_input((1, IN_FEATURES, SEQ_LEN))) inp_fv, out_fv = trace_model(model, hwconf=HWCONF, inputs=inputs, framework="torch") comb = trace(inp_fv, out_fv, optimize=True) n_samples = 16 @@ -152,15 +133,13 @@ def test_alkaid_rtl_matches_model(tmp_path): model(torch.tensor(img, device=device), torch.tensor(seq, device=device)).cpu().numpy().astype(np.float64) ) # (n_samples, OUT_FEATURES) - emulated = _rtl_predict(comb, tmp_path, [img, seq]) + emulated = rtl_predict(comb, tmp_path, [img, seq]) assert (tmp_path / "src" / "model.v").exists() assert np.any(reference != 0) # the comparison is non-trivial np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) -# --- Coverage of every PQ layer the torch Alkaid plugin handles --------------- - ALL_C = 4 ALL_H = ALL_W = 6 ALL_LIN = (ALL_H // 2) * (ALL_W // 2) * 2 @@ -178,13 +157,6 @@ def test_alkaid_rtl_matches_model(tmp_path): class AllLayersNet(nn.Module): - """Exercises every PQ layer type the torch Alkaid plugin handles. - - conv2d -> batchnorm2d -> relu -> avgpool2d branch, and a - conv1d -> batchnorm1d -> relu -> avgpool1d branch, merged (matched flatten - lengths) into a dense head. Each layer also drives an inner Quantizer. - """ - def __init__(self, config): super().__init__() self.conv2d = PQConv2d(config, IN_FEATURES, ALL_C, KERNEL_SIZE, padding="same") @@ -223,19 +195,15 @@ def test_alkaid_conversion_all_layer_types(tmp_path): torch.tensor(rng.standard_normal((4, IN_FEATURES, ALL_LIN)), dtype=torch.float32, device=device), ) - assert ALL_TORCH_LAYER_TYPES <= {type(m).__name__ for m in model.modules()} - with torch.no_grad(): - for layer in [m for m in model.modules() if getattr(m, "pruning_layer", None) is not None]: - layer._weight.copy_( - torch.tensor(rng.standard_normal(tuple(layer._weight.shape)), dtype=layer._weight.dtype, device=device) - ) - _random_prune(layer, PRUNE_FRACTION, rng) - + for module in model.modules(): + if not hasattr(module, "pruning_layer"): + continue + random_prune(module, PRUNE_FRACTION, rng) apply_final_compression(model) model.eval() - inputs = (_fixed_point_input((1, IN_FEATURES, ALL_H, ALL_W)), _fixed_point_input((1, IN_FEATURES, ALL_LIN))) + inputs = (fixed_point_input((1, IN_FEATURES, ALL_H, ALL_W)), fixed_point_input((1, IN_FEATURES, ALL_LIN))) inp_fv, out_fv = trace_model(model, hwconf=HWCONF, inputs=inputs, framework="torch") comb = trace(inp_fv, out_fv, optimize=True) assert out_fv.shape == (OUT_FEATURES,) @@ -247,18 +215,12 @@ def test_alkaid_conversion_all_layer_types(tmp_path): reference = ( model(torch.tensor(img, device=device), torch.tensor(seq, device=device)).cpu().numpy().astype(np.float64) ) - emulated = _rtl_predict(comb, tmp_path, [img, seq]) - assert (tmp_path / "src" / "model.v").exists() - + emulated = rtl_predict(comb, tmp_path, [img, seq]) assert np.any(reference != 0) np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) -# --- Per-layer conversion: a model that is a single layer --------------------- - - -def _data_quantizer(config): - """A data Quantizer built from the config's default data settings.""" +def make_data_quantizer(config): qp = config.quantization_parameters return Quantizer( k=qp.default_data_keep_negatives, @@ -274,13 +236,7 @@ def _data_quantizer(config): ) -class _SingleLayer(nn.Module): - """One PQ layer, optionally followed by a Quantizer. - - Layers with a ``quantize_output`` option set it directly; layers without one - get an explicit trailing Quantizer so the output is fixed-point. - """ - +class SingleLayer(nn.Module): def __init__(self, layer, tail=None): super().__init__() self.layer = layer @@ -291,25 +247,25 @@ def forward(self, x): return x if self.tail is None else self.tail(x) -_SINGLE_LAYER_CASES = { - "conv2d": lambda c: ((1, 2, 4, 4), _SingleLayer(PQConv2d(c, 2, 3, KERNEL_SIZE, padding="same", quantize_output=True))), - "conv1d": lambda c: ((1, 2, 8), _SingleLayer(PQConv1d(c, 2, 3, KERNEL_SIZE, padding="same", quantize_output=True))), - "dense": lambda c: ((1, 6), _SingleLayer(PQDense(c, 6, OUT_FEATURES, quantize_output=True))), - "batchnorm2d": lambda c: ((1, 3, 4, 4), _SingleLayer(PQBatchNorm2d(c, 3), _data_quantizer(c))), - "batchnorm1d": lambda c: ((1, 3, 8), _SingleLayer(PQBatchNorm1d(c, 3), _data_quantizer(c))), - "avgpool2d": lambda c: ((1, 3, 4, 4), _SingleLayer(PQAvgPool2d(c, kernel_size=2, stride=2, quantize_output=True))), - "avgpool1d": lambda c: ((1, 3, 8), _SingleLayer(PQAvgPool1d(c, kernel_size=2, stride=2, quantize_output=True))), - "activation": lambda c: ((1, 6), _SingleLayer(PQActivation(c, "relu", quantize_input=True, quantize_output=True))), - "quantizer": lambda c: ((1, 6), _SingleLayer(_data_quantizer(c))), - "softmax": lambda c: ((1, 6), _SingleLayer(PQSoftmax(c, axis=-1))), +SINGLE_LAYER_CASES = { + "conv2d": lambda c: ((1, 2, 4, 4), SingleLayer(PQConv2d(c, 2, 3, KERNEL_SIZE, padding="same", quantize_output=True))), + "conv1d": lambda c: ((1, 2, 8), SingleLayer(PQConv1d(c, 2, 3, KERNEL_SIZE, padding="same", quantize_output=True))), + "dense": lambda c: ((1, 6), SingleLayer(PQDense(c, 6, OUT_FEATURES, quantize_output=True))), + "batchnorm2d": lambda c: ((1, 3, 4, 4), SingleLayer(PQBatchNorm2d(c, 3), make_data_quantizer(c))), + "batchnorm1d": lambda c: ((1, 3, 8), SingleLayer(PQBatchNorm1d(c, 3), make_data_quantizer(c))), + "avgpool2d": lambda c: ((1, 3, 4, 4), SingleLayer(PQAvgPool2d(c, kernel_size=2, stride=2, quantize_output=True))), + "avgpool1d": lambda c: ((1, 3, 8), SingleLayer(PQAvgPool1d(c, kernel_size=2, stride=2, quantize_output=True))), + "activation": lambda c: ((1, 6), SingleLayer(PQActivation(c, "relu", quantize_input=True, quantize_output=True))), + "quantizer": lambda c: ((1, 6), SingleLayer(make_data_quantizer(c))), + "softmax": lambda c: ((1, 6), SingleLayer(PQSoftmax(c, axis=-1))), } -@pytest.mark.parametrize("case_id", list(_SINGLE_LAYER_CASES)) +@pytest.mark.parametrize("case_id", list(SINGLE_LAYER_CASES)) def test_alkaid_single_layer(case_id, tmp_path): config = pdp_config() config.quantization_parameters.enable_quantization = True - shape, model = _SINGLE_LAYER_CASES[case_id](config) + shape, model = SINGLE_LAYER_CASES[case_id](config) rng = np.random.default_rng(0) model.train() @@ -318,27 +274,25 @@ def test_alkaid_single_layer(case_id, tmp_path): apply_final_compression(model) model.eval() - inp_fv, out_fv = trace_model(model, hwconf=HWCONF, inputs=(_fixed_point_input(shape),), framework="torch") + inp_fv, out_fv = trace_model(model, hwconf=HWCONF, inputs=(fixed_point_input(shape),), framework="torch") comb = trace(inp_fv, out_fv, optimize=True) n_samples = 16 x = rng.integers(0, 16, size=(n_samples,) + shape[1:]).astype("float32") / 16.0 with torch.no_grad(): reference = model(torch.tensor(x)).cpu().numpy().reshape(n_samples, -1).astype(np.float64) - emulated = _rtl_predict(comb, tmp_path, x) + emulated = rtl_predict(comb, tmp_path, x) assert np.any(reference != 0) np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) -# --- Multi-head attention ------------------------------------------------------ - MHA_SEQ_LEN = 4 MHA_EMBED_DIM = 4 MHA_NUM_HEADS = 2 -class _MHANet(nn.Module): +class MHANet(nn.Module): """Self-attention PQMultiheadAttention with every data quantizer enabled. The MHA lives inside a wrapper module so torch.fx inlines its forward with @@ -366,14 +320,14 @@ def test_alkaid_multihead_attention(tmp_path): config.quantization_parameters.enable_quantization = True rng = np.random.default_rng(0) - model = _MHANet(config) + model = MHANet(config) with torch.no_grad(): model(torch.tensor(rng.standard_normal((4, MHA_SEQ_LEN, MHA_EMBED_DIM)), dtype=torch.float32)) # build apply_final_compression(model) model.eval() shape = (1, MHA_SEQ_LEN, MHA_EMBED_DIM) - inp_fv, out_fv = trace_model(model, hwconf=HWCONF, inputs=(_fixed_point_input(shape),), framework="torch") + inp_fv, out_fv = trace_model(model, hwconf=HWCONF, inputs=(fixed_point_input(shape),), framework="torch") comb = trace(inp_fv, out_fv, optimize=True) assert out_fv.shape == (MHA_SEQ_LEN * MHA_EMBED_DIM,) @@ -381,7 +335,7 @@ def test_alkaid_multihead_attention(tmp_path): x = rng.integers(0, 16, size=(n_samples, MHA_SEQ_LEN, MHA_EMBED_DIM)).astype("float32") / 16.0 with torch.no_grad(): reference = model(torch.tensor(x)).cpu().numpy().reshape(n_samples, -1).astype(np.float64) - emulated = _rtl_predict(comb, tmp_path, x) + emulated = rtl_predict(comb, tmp_path, x) assert np.any(reference != 0) np.testing.assert_allclose(emulated, reference, rtol=0, atol=1e-9) From 7e34d6c879abca435cd5af13fb9a946e49c28108 Mon Sep 17 00:00:00 2001 From: Anastasiia Petrovych Date: Tue, 28 Jul 2026 16:01:38 +0200 Subject: [PATCH 15/22] Update library documentation (#61) * Modified the documentation page * Modified config file description * Add updated logo * Modified readthedocs file * Fixed minor errors * Modified notes and code formats * Minor modifications --- .readthedocs.yaml | 8 +- docs/source/_static/custom.css | 8 +- .../_static/overview_pquant_updated.png | Bin 0 -> 151839 bytes docs/source/conf.py | 6 +- docs/source/faq.md | 18 ++++- docs/source/getting_started.md | 34 ++++---- docs/source/index.rst | 24 +++--- docs/source/install.md | 6 +- docs/source/reference.md | 76 ++++++++++-------- docs/source/status.md | 20 +++-- 10 files changed, 118 insertions(+), 82 deletions(-) create mode 100644 docs/source/_static/overview_pquant_updated.png diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f72c324..8053304 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -12,11 +12,11 @@ build: # Build documentation in the "docs/" directory with Sphinx sphinx: - configuration: docs/conf.py + configuration: docs/source/conf.py # Optionally, but recommended, # declare the Python requirements required to build your documentation # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html -# python: -# install: -# - requirements: docs/requirements.txt +python: + install: + - requirements: docs/requirements.txt diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index e9e710a..ee944c1 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -9,13 +9,13 @@ html body nav.wy-nav-top, } .wy-nav-content { - max-width: 2000px !important; + max-width: none !important; width: 100% !important; } .rst-content, .rst-content .section { - max-width: 2000px !important; + max-width: none!important; width: 100% !important; } @@ -47,7 +47,7 @@ html body nav.wy-nav-top, .rst-content .literal-block, .rst-content .line-block, .rst-content .topic { - max-width: 2000px !important; + max-width: none !important; } .wy-body-for-nav { @@ -56,7 +56,7 @@ html body nav.wy-nav-top, .wy-nav-content { background-color: #ffffff !important; - max-width: 1200px !important; + max-width: none !important; } .wy-side-nav-search { diff --git a/docs/source/_static/overview_pquant_updated.png b/docs/source/_static/overview_pquant_updated.png new file mode 100644 index 0000000000000000000000000000000000000000..e3a08d3259a5ab0addb792c5cea1755d68a955c3 GIT binary patch literal 151839 zcmeFZby$?$*FQQ4f*^IYT!B(%m9C3`3VNbcZ5{bR(gZluCDqbccu_q0&l7 z3kaM&;N$Z=@9+Db_jk^@uHSY3@b<#F?>%eR+H0@)tb2B-nu;s|9t9o*0wIu>yQ2Yt zV1I%@&ShOV2TBn6M${09Bqahac^Ca@?P6yNimAb;8G_3Z4!Huk><@tyKm=0nW1bd3 zAQC&jsASKm?EX#V;$$y`rWE2r3Ub5w1kKpZxDZHo9;i7lyP!FOhaJIdF34?$fSZ~j zc|l(oUw$=r>#RAr^Z(K*2i(~i>4dU!uy+O(AZx!WNSszMb#VG;1tN6c9B{-bHHRw~ zx&j%3%h{5nv*HS?ILB-5!X`RTm;4P-o`V3DpIQ7O4#+Y;yR$uQ0^x*&qmX8H&JN}% zH@Fj0!UAcJvXp|`yTWNC1$?b6PAIaQ=C(+C3zQ|fB$Jsv+zx4v?$Mma4sLH{4oDDiTMKh%OE?!VpBcKR zyEO9S+duII^`|{-zvCNjrgv1{TN8bnp?BVvm6TaJ^>sDE5pE`j9?RKv?&7a0Fr7|; z3t{`u1v?ch4umb-%I<8rFecF>&9k<$2g9>RAm#74v$@+j3Rzj12_Xg(^V^sBR^#7FfwrmK@h zFV!9obL?GR6&TOyxr20>XCQrFsHaYAX@(F&S)pu^NEnBsy~RJo2LchT`6UX>zeM3b zZ1&#_0@5QIik5}pV7;pW&oKlT$!-Seofn4&o)?A&V$x+|p2IkI?h2$!;>wddl?|8- zuVQpH2t{L|1lYJN59J>25nx~toQFVZVKh)`Y)ow|Od?WEu$Wf%7Buo;wVmwHTMUJy zk&=K?lM-<8LHW73xwv^@+yeK`9=Tz^9-#uTOHh1t8+b(LPj$dw&lmrCz6K>e>x1A- zNBrsu8hDQOp98`;55WvPM+gDsc;^DoVL(ds#O{*quCKQ~Euxg%x?N;f%ge(}z+FV# ze;9UG=Ah{M#ZEDQ{F-bo?rZ|XSR8+S+t`Svx{hI@cGu^u%%QR;7}j~M6{YR&L7fM0 z9|uzNYv6TZIr*1cvwAub5vy^}s(!GsYw>*mb$K#bAzjKJSo$b^N^|v&)VlkUlV0?u z(6{J%sst#OMbC7lZ5-YE?)Ef8=TS`N?uDo2QBXybZD}K>nF@u5@;dL!tpi$2AtvdA zBQa3_QthBbwde~~l=NcSTttm}-sdjF6IeSJAE&S5L>=Ze)-Ct+K3yf^#YptvJja_P zo5%d<#y-hmdba$a|F*PyI-BCdSKl4<_dTt(F1~vfltuX3t`WC`;F2MGqqxdRl0FA# z(~Dc$tNC?RQXz{$0&F~$jyg6vWle!O>Ynn&)V!xM-6F-_t zbbV#ifwR23a?aYffKt-@oJU5uN;`6w!-ucBKC4`HZYCh((^{k~-fdDtG&gBt?wtVgYmI zhBczYi#jSsgyM#x{Muh1KhIT?P)XTIuR4c`0l~=9foef70r{aK#K1Vg!ooa<;}0c5 zKQmzAL-DXLuwrAJKaYcnbt;5ZL};ROPzdw}nuHFE40_F`#&WQYy}{B7!Wez!DXHYC0NZ-2j70w?Ej1&vm&{o*)(%cChff4^!7gE2irHt zw-=bYQgV23TKG#`DMot_g5O0D?4=gA9aP*A;qA~SMtY0dKavyT?J97Xs`?HmYlT~V zczI>GQj&f8wcy+{r|MPdkKJJxGw_YyeajN9?(hgIP95B6crQ-7_J&O?p6dad-GcoK zlPB|SEJ3&FFg0>!N6!0~T(HUx<%|4jEkfj#f_O<-J6ruAe1d%LLdjB@Wbjep@(CuR z@SDqFYPy)y!F9Q~a&ESPwyD+!Ka6>OUXg$DyRz*lZg5`DQbVK;PfGhC?Tzf<_!Onp z1%dW%j%G*KSJwm>Z*y8-6<6A^eB~4n#(HSzrF=qb0#&Z&FU3i^k=33#!<#3m7sZM!6g= zt)IFM+7HLN6R1O2Wy)n!t;aS}i`n$J>7GVnu}2tB<>b<>_{N(-MF42Pj zZrf7n{V`nsmQ0!wsj39aRBrL*l8OruSvb~;Q^LUROV=Li%8CkKmL_IoRGRyQX}oP) z`_l;(%JB;^W#4R_~bxC1!X$($!uVl;E%~5?h$1Nodo3>D50a z=OR#|cCd?3T(pA2CORjf4MPhhc(_1>siLV1BLvjo35N(<)5FXj=>a2xUO^Y&5#gOF zLp3J{GuY)*3CFrfg!4-`!hqbO$uNls|I*7+5;SPqzw|MT56a8U&jUNt#{w{3@F)O- z-TU9t>Hm}F9lid2rSed&%qnLV>UE9LvIeVuDq{`k_(%XiBl&mIeJRpJi_ z>%oDsn#gp7idL0SVoSTRzd~_>VwK5$m5f{X&i~tc(cRKfsWnS_A1(UM@yf+0B8l3n)7&mMe^bdt(QN-oJhmVuD|9Qa~cmq+BGThA)k5Z;g17MOP-q- z6qzUk9=wTCTZDV8!?`urB58OqCZ9MMzGw0Dm<}?rS~c<#Q>*YMbeTik?QYLlYxNNz z?@QcJh;YL<(jyDLOfS*@Iqd3Vh7*3Z;Fv0d!B8Fvi6hOL22Yl^gzq>ka6V4U#wLaO z?pqCSZ#+!d@Ukq&MRUnyXiuM8&8|Yi0IiXbOvP)Rku>P~M{iilo zFt9szQ?qP9+=82}KKydC2WyrJ8Zh>kCda_Q3h0IgbU>Be40Lx=xU-6NcAPlxaiuMa4!Q&Z=T7_m!&&CB5NUT^h({UvqsQM)lYZ`e*cW-$GD z6?HHZOOiq`hJlQ&IjsM)Y4Mv~inig+?}VBg_Y6N#U3)Avqr+dm7{ycdfaNV`W0!XF z&qc>)PPsFo4XgAA-ZCbpJE$w?P@8YA+pbd>f$nBJ^R0Pu!&5g^`TdDR-UchY($| zPAlQEXiH&suAL_Gg4Zs+lmfG*_?y_JyEpVL?m%jen{&4eu}$BnHvG_BylYSu-qIpG z{%TiOz4qbQbo0C=j;(plxi2rGo{p4iE)D94u_2UqujB4)8_IsG8~c#C`6fWQF>dAc ziUFlmhygWh@ahQZj^)PrZr*^p>M`~Pdl{8|FMZsuQ z$s8ZZ6E8x|;u5czAxBl`JgmO5(n`(??|a2P!F0Im;SRg&r1Ml9E9Yv0X@I=1XX{G* zfwi?8+a%lAe(>r+0!5bS!8Ixg%>pXFmZG{X!igBMz-j2y_s5@aSm*SdlhN}|=p3bf z*Gu-KPg76%r(55Z!gCsnTF;!O#th15f^pL=&J9-LVaiBlc+9ws+VRk8PHF{9mGV9| zm|Wt!d}(@cyh&g1?b`lSG3Ige&v6;7gyBz7HVvhsPRdvxbg?xLGK*PoR{bANWISZO z*rVx6&V__rE5N*d8TX^L(S7lK`YZ`OYz1KIWPqvT_#G@%*tTnm;^S=HbP@ih2LJp| zJBJPy0s{VR=>D=~`2VzIq)-yH-MWZ>Ny^s2#f(PH7LGD^aI*Vt7V%HbA_J5TO@@ho z{ckHsqv7Cy0ucv|tgVA7+?Ga``?vMP|7AUY89!cN?LYv+%X@14VBq@o_(RgCIj+b)JzEE$8`Vs)HusY00uy;g*{4 zOe!gnmNBfc=HqPXe4K|m=XNy=RGqGyaqGE+%W3#dnpmhREk%v~j$4^5($ysk zUb=j(Pl=Mu*git-+8usli_2naZ_mb@Q^Bj$iPd?tjbY97qoFrT_EyF=jN{h%L(Ay# zD8lL-Fv)#y)wsmZBeyI(5vu*=WA5i-LkTm z777+zhO;yNID)qwKgaDi=FB)biI=|}xKd=+>Ll@uS|S&V6SLD@%67En2FcaY_-)!G zW*(+I{h4xw6;te4@y(Z8lmj0ZV^j@dq0)zrWh(D}4(u0)dDwYj3)~h`MKOI8eYNLv zW=8{hpaGr#!D?dsD@dsmF!QiVptpH_9DTJkj%Rb=DdO=z%<6x%n*WN6Fdjlp&rFRz zR1d0~shz3$Q2iJ3W(ISCo7qDht(~nb1?&K(1oo!6699HN?2#x=w4LBZ@(7y3&CJ-L za6Ud@)VSg7rrZc~b{L$;+ze{Y$IWX7;{b&JMJt>dlY)pU?7YH!WBr^pQbEU`lCn&; zB#%cw?7W-hjf;#6+w*mDEffuvPMeumOF5Zaz?^VxIlhNA9K%HCemuW(oV9Un`D?f2 zgAR`;_@bhWwn~=?%v3sC-FH9xB?e#{@O58exae|Bv^A1(gTzqayR8G_ewK`F%G#$-|uq3)rMtAWmErYE8HP4DG8c3Y=ycb0+Uq0;_7R6HD zUAJgKGAuZvXeqOdg6^3N%+IrO&~$CiOv*+)(-$<4O&7hPd!(N7T&cNKK;YZcRhR>A zL(i=NdOA+W8Sm1ZECI6w3Cwo}ctx-grG7iO-iN*c^3|%8hMM4eyLBI&EM9&exD(s5 z^}g-Jl^layS-B(Uo0Cr2;IMWi4TA4ol|YbyR`S)2m*p!2mFv&Yu%;gK`d_f-pU*!W_``ue z9Qeb5KOFeOfj=Dh!+}2>_``ue9Qeb5KOFeOfj=Dh!+}2>_``ue9Qeb5KOFeOfj=Dh z!+}2>_``ue9Qeb5KOFeOfj=Dh!+}2>_``ue9Qeb5KOFeOfj=Dh|0xG{_0B;kehTIw zjM5)}$lWE?moR2({XQ>5Z3nYLb`eD7`P6@iL_hhru!*!*H{!hQ)4j#8(jRrc!sV_hAJLrNf6tuX zMzwq`P4=yONiOwWvGmx4%mU5E^r$?=7iBH-4>_K{3;%rK;YRc@gPFVI{qyIDjy+%8 z8}RR0e_~-3lndDSu6~(`KcWpd9!6yMT4UbJA5O%04cArDjmW~uK-gtg6?V%?RJqL6 z{Xk+(e$eE=X4QJ_FrRjJ0ZN&-mr*VQAWU-w4fp7|FI`mhP2bAQuYg|_ZBK5!l7^uP%cms-((81bx^C?OE6Z&q4&QFoPrqA^8USlNp*t=Ba%(O8*@GU@UvLzNw+kd{_*o=!+jPZceMrwu~T zj7eM!PsCjaG+>8B!D-y>Z0((e+(nsA+Z6)$=wePLK;&d*E~Ifs<`)Tg5@q`J7Q369 z8;2YB>G2UTK|w)IC>JLe7dxO}clH1W6u7h7JJW#{@MuokxPx>?IDzvfz-b0F=r-Y| z4lXEBCMGfTD**LnCrVFC<~+jD9@p zL=$=c^x^Eu*6Oq;M>sel0e$j?D3b>A>|Be}7SNS`H6nF(hK3jyl#7oY{MeC;M}+gg ze1-1g?+SOoAr@j@jQ3p}P{L*oh_f4^vyVFxa_W*Uj!Nd-4rc{yXCE%$Y>sy zI%})DxjI<zoP(W zlsHRR$)268fiy!GfipACWDEVf#nWFVp2`%`-O<4bbtX~&eNCbN{wpy(q_YU;fB*ge zT~)9(PI>VEYMy3@-+RRo9L90F56uvqNLwU083l#jDX?EV<*#!t{@y1Nwy1ya6QBTu zY~l76qD=1WW=M0mi!I=Zl!K!Oje|LjvXYdHv6{LI+#aQ@M8i%4`wz?bt3?sce{}XA zCU;f@^a;=`oWJ!Cc=)ZskoG`rIRX9jgR{m00-=G(-;vOAf4Y)}bC+0lwmmx24{8$H z;8W60fmk;r(tr1%ZTd;W-n3VwAse#>$x1_(oH~{|me%b8T09&#lc19&-x>3#qW!Z1 z_G=M)KE9EJDj%oDr^d(kT+Nk+%ZdiV;^Hz*_6;Mmdvr-STQtS8iHO%nh+J0&@v@n( z#D#LHGj`ueuJ2KjN@gZPK39sfdWWRw-nxLk-50+^t%P!ns!3C-br;-E@aoR;k6gc6 zdYnrraXDKt+4AXDtPnAlbwiqIDInE{DM$K124otN7MTug$Fr%;^Y&qS{$r5YI{+Hzf zD|HyvqcL|McXLT#Q!IUMIo5qst7W>OduDQ#O02cjys!;4m7V;Ki&^vM4ThEJ;uYDX zxAI1N^YZzV`evtg4E^5)dfC=a4r6U=O~#pxPWvp3KpN(&%i5r-FSBllVRNN21Dkrra-G#*(m>kC)iji*8{XK6(&@6H6^V zKi&U(yr1eik@=-%i*1RX|h z%~m?uruy}d6TUjksFgclF<}~}YK+Ep5_PesvXJ&y^N_(yT{p1YHk%xdR8(sD>44~1 zmSE3!s4j1I>V{TUp4jDa7%j}8ely^cnl{> zH(sSmtL9JnyLq9|nf}#XG1{<17$Dw$B9VOXIrTl>mdfuIWx5ijoi5!tcH6+MLs->6 z#k~;dRDaCNtZ5v~xG%aeedjmjZJbWT4E@SDooAd=lkD|XG-WBEB#NBy+3#z8BXP`? z;>C(S4(jysUt#R(S)2)&KHT$Dk)K~uyL~whv11&x>0?PIx)+RQ^goFi_N4-!^flQx zV;VOyp2w09)I8y3M!vK^Q4#sP($W-)cGpg?+q%yTO*lv{?%D6~3~mJ#<~pW2xg~5opVzi+;b~ zu!JBct*Ns?`S*1FBjhi69h_F-xRyotu+c91*;UK5@tPf9;X+__XYlcZU@x1~dfyG_@~zUve0L){`0b_}?~hcn zj@frKv@u_PJ3Ick^J3!5$MkvL3yq4v_kJsna(XC%D{-0h4e+HQ5dSxqf3D0wQb&Xl z-!i3D?@MW7H?CPoV~amjImbI*>Vo5gJ;RvmIKu#ebbju%uVqSiWT3C9{l$(ziBpk# zYtmLwv9Pe9VH2a1qtJA{#Cfi<@HUl++Bt~-^yV_R!>edN_{_+FxRaG6Oj&L05x7ZqMk|dXpd|7W&ib)Mpuq7*)uH?M+iFSCj5) zYL0+8InUT-QlOj6tZL2-rjhITHJFC>G4a87@8VobgUR7Wx|wM+>y`M#}B> zg!e>oKRD<5iwg149 z&gJT!yV3O-+Q_bKx{0%n6Dd;fO%yX-Xl&d2EjYDmJ=!39j=n9=wl628Gs)elRIY?v zyILqpv1LsBa?b-y$ZeTUgQ?eH{8$nOSV33O%sWtw`J(wI@VxZEP)m#C0~~(8&sSq| z4fZ!hJ$H>N%zeI>B@XkzUJ)e#i}2h`exjRRujr7_o17%C`~#oe$Iv_Nt+7!qlzdgz zQ;cqFlajlxO(*v^HVJNSd~7Aw@ZHBFX>Fn8v-aLjBGNSX4h+1N5UiBW)u6RJ{DpbP zdbGe{rbM<@+@jy`8n;T4fHdQY+;K%|=~XU1>&dzFJt?81m!f>U)<0R+*l*QHJ_tj! zN9ca;rFIh@$@ylS(-Q}OI7&#x_vLd|-bDBO(u#s^pvGdH0pyVI!||>@du{KRw#QX= zdxwStI0|lv-KYDe$sN}X3?J`9WFW!_^_PWzYug?J91<1f5c974(_YL*#|?-k0{ z#Z3-HcnYjXtrZaDg&Qv3SE;^zyK%Qp#W05)hmgEoAQXr2V`xe7y{cpZE0$Pk5xnG| zv(+21TtY^rZA_}hP@L4Hp8;{n>epzwxw#i-nJQww)j6w)efNq{uSoKCDIBx=z$R*J zSbNNnJO0}0+vvBo2Y4hJ_X4xZSoc`3g9+fBP9R1;CM5B~-uuhwY51Fiy;Posnp_QU zmo7G{lXwus__g_2mXeH6rRNR>IkGpA(->@1B(aUXJvA9IOF|b}Twh9+T8Sc}Lpggd zWaSx|cKsb(#`~bJ`Ftx$g>T20+Qf=l)sFB5;gJm@#=?Hu>pkpl14WA(c4|iT-a%&j z_9a>E;mCecJ2lsp?YsvMq-8uF6W@x7$Cw3?yEl4!JxWxnuJT6@5JC>K{UJA@Uh6s= zVLorr==oUBDo=0gT5O)u4SrvZSw`jM?~)Jj8ZJaW+v0kekdrgN)YLIPF1$YF_X3{P zuJN&__T+YIkgV&vXmfQkC7&Hhj~Tv##8DB>bph17`PTepZu1c@8*}r8Z#S41atIXE z85H9SG;&z>5z{}VB}>89AHhxGv8#06^CcY{(ai0?9O6Z^vob2nCU`I^EY$NQhqd3h z{@Rw%8%`R_fyN-|Fybp&_sVVK+YIpHv61d`Hs&wfM+-(DMv_TeFAQ)@C24wm?Uk;* zJCRBJF4wM3+vdG+O?5O;B9kh$Q1|W&%)Q6xwZi$gur(ZRXRXqc5YZIm8XaMy1V5gP zD|c|+&^;b)V4~`&He^ysC&^m#5H~X=WxL1HE=F)EE9+t8b#@tff^I=)(m~1!HxZQ% z*VXri;V~3(uDks$y0LDeH=>@fG^$L=`Sp)KlVVE#nRPetTbDCxw1LC(FnB5nf2@Fr^wF$+O={)oA+rVE|S(Cp-ZVYNc!1cz>;NWLm8 z0mcj@T4sG<`Y2@S!~Vr<&P|;dklTL#Xj8!X%M=_BSLnOt=-kV#c3u}Ve1uYiK67X~ zMp1a)q86%mNv%Y2cvz=Y3lUt|KkO40O#75*%<{NoyDBVPLN9-qv)q}f*?Zy^qX~Yu zN?M&9;bKA^Q<_X|x)RXYtsfYAbnpTeISg`~cgm<84_0#4Upl?i> zRFR9i_1*5H+mHw0I5klgSuzZaXhLX{0tn3|Mg2~!{?6cd}bc};O6S}-a84{ z51=>X6m%b|dmrsasYpdqK+q;ngGKPTwc$Wc_>IyR^*N;3KJNWHc`9k6#G z;FP^^5(o6YS>i-*(1rp!*9Zz?F6-XACBc(5&aBEr8JU^<<<@4&Yk42}U#XLqesZuT)T4J3g6jlB%xWx4Z;hbTG+6amqy0!u|2b`aUw?! zWQaE<*>2CwwmN#^Sk7u>>J(3~OumdrJ0?bGlf~8TevsP>*?%!Iy-|I+pDb>TT5Pz! zcQUO&`=z&s(7IdV9EV|UZ?bb@GP`vYD_xM&x5_lskF1WH3In8ecXVDENm;pXU1QH7 z_d6_3NP?hUguyda^vgr=yTAASSp&<&Y;395Iz|km-|`; zKTz26bl)Ggil|GwlC|{RmWz4$e517MYRR*aNAIZ-YAl8VmK(z)rh&jd+8CSw_ri5; zdS-v%rN`C+@4n7(L8tA@aQto?{f2+{wA;^NcO-6pHjT5^Rbjl2|E@(T@mVg{GE*P&0AU3E-^W7;OmI`{EVP>7VJjkM?PJu(27iB+?lD`dWYmM)d2s%)HZ)O$Eu73 zy_lB&Uf49?Yje9AfpMhp*@z*(hN~mm`E1`Mh1w03l59zQi7-4|!n$_tTG?W+o0#u# z|FlPYuV9Vy;Jg}%eN<%>d`X|O|lh37A)YS+sIy(d?Di^jKldwZ7*J$Gp}dJ7{Y zCAqn| z+J!oWfQvOXf>hJ>(mlc1dwN|tYF272^yyAA)*sj07MkJch@kzSo46VJq1KLxJ3_yp zT{&fVM0S{-(JzVL@5)gLJ>As`_vJA9HYP^8G#PztQ-7P>_x-#%zo&JM+Exi}rSFb#Y$OfxPHWMud{z-8WI+yWj6QDLD~*}~+WtiHRr z`<&Ak;=L&d#EgHh{sy&}X_Ix|!K&~s=$iM(xZ?djdl#C}hPA`>4;2n?bBpr^jPTqy zm1@fsD`|O|AAC<#zdu*aXFsv9Je$|n*T%%mVNly>6WP~2`prN!{o-Ty92Jh3@VSZO z0ecNa`oxrqY`vH3SBbCMTdXVf3s}v`YpN2GfKV%w{V(kk0DBG8^upUPuvE-PjhK(f zG9>mzc&s}IKLhExNXj|e$PMUIfU1Cm;{sPBFEXQHcH>EE<&~EPbyRzN$%jT5qMt`8 z$wiNE0Z(tOE>0P^pNb;A&6$Hmr0xTU&+0-VAq~MPI=|hT_S=sWinB4hlXyijf}+Xe zKrcjNUJpRnI_L+i$SmWJ@t!ML?ZhlvauZU8dRtYaHvFGL%s%;!r8YvrjQf2OOL0B3 z6Lo6yx93{G9v)kpZaH;_LG7RpP8zw1Jzn+7BBo3jSC9sZ2`%9G=g8ukAL)V6w#>gy zE1p&NzN63^A6*Pt6E*8o-4U(2Y}zUAI}-$7#8xH{ zLm`9b92dQF`r_(=QA(FhR;7v2xADR;*XA?rJ{m`UFXEz*xvZ|@s%G?*bg?@ z+1d3g?4?*TZh>h^=&G10iQp!SAZwGNuWMCUkDiRz=<4%kWdET5nHb0Ro-d#OrenuT zpGi)22t*Cfs2~sPEebRvMm(22gWuF!0O2}!< z_$HxPcnQmaNVD}t>l|U9emPIqT`<4?4pjwymRP^*B4&4GYOKfdDT%N4n~@71IKbXa zdR+YWzEIVoq`cglDkJ+qPP%7$_tvK~AeKPX z^SUG-d$8qLCjH_1+UHpHHkr0YhQz2a>%{j1TdyrkL#7F-Zyo8}J|BshF|*@++mHTP zqXhW@mrmMIFz}Y6)bW5AnaDK-96$2~@g6-?WV7PO&710VGbf!|$$Bv64%MWh7B(U? zdwZ3GQl(^{gL{Sj+U1r*7M!%pY{0R*g2=gHdx@HBrB1dWS7JQhIT1L5FFwF(>jusO z<@;(tarb%ly*&lMwgZ5LIapM>c<~|$ul1|(UEMId6EjniOGM%gQtwMwq#o!*$I%QucHn$yG$2}nFkwWUnPy< zrjW`Xo=l-xJ0b0Jl31{+I0BLXudV%Yh%_6xP>r*1>^@sh5#z)~n3McKG~ z%Dkyw;-J0q0r<2+sHHegW3dvku3Nj?8=5cRYB7lv!$%w%*^iRKo9;U@AS|y2dc5}S zP^q>}?d@TY9Bw|FY175EBY?H!DWypW7XvRm&v^(t=LH3?FUbMx9CM>rwlzgupd6{= zXv&TXhdml?_^I$Q>yoKVI|#Z#u(M22`&59}r`>i!jdA`5fZIwIV66b-fehl<&ul!u zF#-TU^=JW>T{{rJli*VS5bd(OjIBOh@4;_B!PqCA2jsJF3C9Eg3xSd?@(vPKD*%c@ zCF_w>IemYfM)tA5B{eiQmrD;o1B}`4OrTH9G2Kvtld>D5_-Vl3;QnrWdATBnVCd#^ zw<3z+H$-JoMoYlf#jRakW{LI5?YK8ox+bn>^*I~IsNQ`e{L+@V_z7D&7vF5Cra}j0 z^lc}D0s;j$!W;R&1R#$L|QPZ?BF#BDBB&%KswIea8HZH`8_ffpX|j8;{dt^sRf ze6q;nBCNZs?TK%>g?0Y*Cj<&Yh}8*~Bm=WVg~VGRsK`q$TUZk}Q4~^=bdUK$w44!2 zF3%Z8Z9EIGGl$2n_sD}vEr*_i9fdn8=_$wJd+nO~Or}|52R)UWkn3?^n`zv;Okyz` z61eB{spGHFJ1L~iw1R-C5Lum~dy3+@hN|f!)h~eBNJ>pj^;Yf0p9pgx=>9rtV-^Q9 zZuCa2KaK`I`)UOeiRuiHl5Jz2P`|46Oj5e!ZSETriazYMC8uga_)$AMOKDU=?n&VS zXh(Wq7eGHspB0e>pd%ex_j}M=J0X z-;}upY^Iu^&+gTux!f^vWu@cWekDoyIZiy4UHt&a2-&*-Z#46qLr8<{S+yN_h0aI zbpGDC)_{}b;w6_T#m3E=C1n|;q1C04Q=pY^EqU~)@E2d)t5t0cOiN7Ll01doKlFFC z*?lcf9v)!fg)*N%8f{nvVS!vpwzt&-eup6^;C@|FVFGY<>?I~qc4k0#r&?-8taH=Ia%;^NpvvIa#Up3=fw|g{XYv zv(*u*xe5oEnW<@v;m=Zy;O(ESSlNdRfWtYF4!6x&)s%xu0q`_jFP_NG%9xqgY$3EU zXDFGHxb45p^QGDQ6h!y`EV;+3)OhK^`=aZ$iwIhBer3;#0FAtg=+S*~Tgb$;sH;k7 z%~dA6cJU}G)UebxhwM<`yC4_JMSG0q`RXZ5s{4rOrdPGDV97XP;Zh(b|+hAU{XA#Bq@2aX4t65p*yd4sGU3r}l9k_>UFv?RK z(N5S+Ksv>qo6Zbp-30NS$^j1#h`8QvtGr{|uj3of?fbZlg(3|mpKN_0KVl9%7&Atu z6kf0NBpTpAZL_VM1Mc4yqIWMfb$dI>;*t%G)SgvJ$k*Bo*SBz<1U|e(dGjH6n!ASX zaD86CLH3A2!&>@IiZ1ZOz#gqe!*K_&BZ2QV*bh4AZdjdcj2YV~gqYfXYSN<)Ap5OZaD)0%TxfFW`|<}^K_!Sf+! zWgqG7RjTbC?A3E#4oC4<0J$+0u(P{3=reonJl-J51vG|~6&LE7$_+sUeI>VxQ|QZa zB1=`8z|{xQX_|jYSEF#*zyPu*UC>QWX~5e)^WHqG)@ANtY$Ql*@V=1ZCFovYzo)P@ ztY}>TW!@XVp0LpHZQoyI*6B-}jy3fupwb+DjEOxIxg4%@L#L13O|>>k^zf3x2ymU3 zREHOO4QXVCJeZWz+s(2`S8^9QNHdudv(sA7*iih@Rbk7TFG;2>Kv;-+O=_=Cj!^I3 zX1nsoCDEbg%p(0)Vnm9s<`CxQ?I==M<+`=G$<{NBJt!Q7T>`TFO_i7zvWphwh! zzfdIyd^D0#Y*OYv#ad(2b0paFz$4H0P)%>Vop-pM#wvXQH+Q{9qxM zHB&hJ%}OtTTOaV0FYDC1-``;k;A3V5J37evU4TSB${)Y%g1fE3^WS1{Wx=L;hqCI0 z+;wuI^-m~C>kPZ8%21>3NVXD{-&6Hm>BT8SvjIK?(pRJiR-K~O*5P9|JGEh(xI}hcjx1#m zbQ>!%s%R(++Q#m{C+kaNl?P}ZMiS^gZi>&Or8Y}L8@Pn$v+3bBBz@Ak+KbHr+_iLuP z*$rT}TAuqtl+huP=`ragC~>Qx&8it|E43#=P-ex%TXejR%Yp?UUdoM6(e?hgmD=Qp zyH`jI^hc-qB)~9yl}?5lgTh*cU03O>HWvpRru=A@(?bVPoBW2IIjsEg=C5WlpC#p< z?&CB!V2YGDCNk-dm>G4HgSa%aq{xCamRlq!2y?G=zH6rHbF!)o>0J!io{XdIP0Ry) zn)2E-#zV0V*{1Cv&Ul3juz3bye5)tHh?C8hX;EAg{CSZ0KUF*L-pxnuY|38H%k; zB?A{33-UZ6D9hmH374;o0Jl?k2@t0wn}hIiNt30XyAyhp{JvMy=8;-Yb8>RJ#xk;m zfmC_DlK=?4LeAEdHz^fa9ODc}@r$VKJMv6i-*`E13$+e^JrCIBZy%MH3wxNjG$Ieb zX6h>JVsydcJz~7>Bx-nNTT*B{I|?-okXdeCp@X7B2{N{65fp+4>t54@faRPmh%<=^j{~64?)r+o08vVSHzQ%N>Nd)a`}3e0EQ&YYl=9W#~c>H7gICL;^0 z+p)C(v|>)f?EvYpv?7K0f-g8W^bOxfitYyT2ydm%A7fUU5>ursVRLlE!QsSn6z@1Bu>f^O803s6&#AKp^HWcV;E;KPm-D%hQk_g}1 zsX)HW&^B?BpbZBuQ@rY0)=jC9CQmw|EHtoY76;Nb__$5}GzS~n9g=HAUqWiF=nFme1vUqJ zX^PSCgK1mXEz#!y##YE{AFE!#1#V%&f;I3n_8wn@K#V$=qDdXD{o?tQ~=! z^TluLd06S|*Gb%N7~`W^9Zx&$7yHvv`#ZgF4gqZ`v~E1*wKned9l+{l&f&sVo5d3H zEK!XUJqfH97VE%9s6OvJFEd~uKTuJC#t5UL(P#s`+MgQ zWL0puv1Mz{YW=ERoW<2-nB||Q6BfJckR`TOqv>ykkYI1 zOalq&?8|HMQMa*qodJ|RBZ=%=OCjefh2_v*aqOL*oQt* zG-$f72qN+9hl=yv+AAw*P5P2cJs)y;^bEs(BR#X~o( zpO2r#?(Wruf{`C9*J;Vm+h`2j)c));;|JhR5a2i;tT6DhhJ63}r1#Bo>%kL!0Klh> zC;I`z<=OiNxq6_Sov=1pcQWO%RfhXJgXI4n3=%SU17Jyz(n|+?GpXF>844Nwzb<%spuN|AXWvCju>}By2(%N3 zLd^Sx`{d+2Rl_#@t{2V5H%b||PuKgbdsxOD=yd}8_`1_-J5_1b@U06NgF;Im{hwn0 zPQTu^$s{5KI1;1M)8Sk@Vb6xZmIoJOFSmf;9v12_-8fb)ugS{F3cC*?o|Bc~Rq)yY zN$Mk2H2?{u7$wCsgGdep&9V2MDiCyoWTfv&R)avQeveKuIu@8h$6@dR+^q#p+M7eK zrj?)aRJf+qO3H9^mrYcBV>7OiskINvDl1G%%tV>6-O3TO)wotbPaZW8W6={rryO^)(uwL3}s>mad( ze)+`+^QMFf#D_9!i6Nllt1x zw^zLJ3u!za6lEC&i<$K7yWmAD?t+}=Of{k4V}ZDrM$334Sf2Mo^~x4uB_P*-vcGSU z^RX{Yc&~u>Th93jVd{Y;9$d5?G5>%yj)kxA&v66yOZ{1{L=9q0j3hmgCXpyFsU<_i zM;+wgM{BxuUt9(Nac>J*-2UdgX0%(+dxrx=xrE*;-yo;?p2AMAom$~E-3kdofFF`b zS6ELyJbqP)-woWKzP>(f+eAx%?4T0BJ2U`X%@n$*Ue)C3*!10B{an!1IiA1LQ|RQl ziPj75R$@QUpyGmp-8af5K^_EAbU3_UMJ7Ab|Y8*!l{nDAcZ9 zNxBM{g;~@QiJ0RE{wZ}cwc))NGPEK+l;&UToiy(YPK2$v}XRLfT zAz!O-U?4Ph`{64-Cm)1Mzma|=x>KB|lDa_^#}GPJpmFKo9!-9^_Tf3`dH;B+3jS8o z2iRRrwWrxx2@x&)*5`o}Z;HravRhQ^JnOrnL^0yL=Ce{^lsxv;L%kEiG>^PdB7z1woFn289Ka0VNT_@L8;ZmZa@h5}#LA z77q^uQHnSpZ}0&vs=~J()^KfZ$a;kG?Z#|Or^Eb^12@0*Z2iJ)rPZ(|_U&?f7pQkJ zv9Xpt9gvZ{zcJMPznw2|yhvgApjZdB&7?;RE{>z9i&9u`JXboo5*PF_Q2X$mg1SKf z9FN@B{*_0mzk!sLyp0^zc=J(V76B1%sHtP4Bs-~Y5as@5k2_8QiEpWDj=_&TAlG_a zGux@U&HJ_C5fXhLoU&^!D`p`2OrgTwx7snB@A>LOavzb}?}rT!ik5XT<>}#_7jr z?za|gA+c+p|D!A!D&e_{z`&+FM$n-$uMl)2cb}G#5q0bm?Fve@RfL!+^zNr8)tD3p zejSmEJe?AN^P?2AElkikIE+-xv*jH(sDXz%1iW$MJqP7}H5%fjvM~CV0{v8u z%X(O*WM>%rZ87<+fSBYrr`rtB3tZ_<7RY*dNX9wuISMQ#gE8XM3*1nq`iP({KQAtLiM$zlx15|=33Lrj#w*wrO7oR#P;2WK|{Q>BrhTgvK zO_R;bz>#KKLY1RQfqu!I#s4GWbcT($=%nfl%-Cnr|)Of_v?7xy){`>m#_&s8$H<7 zN~-P;bgRGO0|iY55U0`W&SA?-Aq2uU>yNfUw-O+stivI6W+0oV{?fKAWsF zU$-WNT+DdOnZ*$FjgXboPT3k^Lm#ey)ND#5#*3ck3e?MY7EO9CG4nn9_XaPgFG2!n3)<_`DET_gJaABnjFBX z!3Bgse2pn-w`!FM55%Uv*$59q)33H~a1JJXTMR0UgjOfp4@8E*VL;{cp=l3 zbGU7&kA+~OtGU$?s6KrIv9q-Di@~8KJOfML`#F9Gix@u(p~l~Jz5;4LevF8L_F~8+ zT(`!RntBwpu^z8-cC<4`d+C&0w7tv5=>qla5q304Rv>qfQxKJ$39&=;+!lU*DAjXy zNdZ#as9M6N(xcj4gC+3;P2maNac}C+WPV)~M_@g?W+umi@19|0_y+?f0k>P6t$&9~ORs z&L?P|YQBjXE!OMk5RlL=1#)T8odiU$FVW%K@h}|V#xVeoW9s%+dko(2m2x{I8YYLT=KUm??eQRpU#_|&=BtcT%acUr$(~ea2jkfvG_YJTo z3ZqeF5`F%U7IeCRc&;NnovFws6KIzrU+$2BrYAWDf(}|>?)gLXB4sV7K0F4d0FuzJ zijI{Hhf~n9HceWB^#jE_wV=KI@`eL2C7>jUEoLs_T?FL-K=l{zSM^n^zZQqmFU&93 zS+%aT%5Jq)bT^qp34v}1fU7s47hBBQ&wVecuobK#;z{)vy{6!lB;nJY;Z0&AG`Ypa z`x3TMkBvaf3sCG|Lcomn|yi@M}w_hWj zp(CoU$jASkYT33uRh3D$kVmAw%3qVVST7bHjAbAfmXg|=DU1a4h|Os6Z~wai}gGyqwd&} zbDizDqDgH$avrzYOzJvIPTj`NYZ-Kz;I9d>F^9q2hV5ZgPi*~W&CmAcV_20Po9rUo zMv(#Q`e*K;i);Gyo^Fk?VW5)UmxJZSnOJ`?^CVfI(=-n=Et~1nUAUn`D%khL|7z3O zC3H-=)D;_gs(~trUgCa!c)Dfju@+>gpv!Ptcb^>Sk3Rk*n_4#UJ?IraKS{mZ6tGZm zq5yq>Yq!fjRP=}hA-kK6<0(sv)P7yt(<7X_p|MSIIlwG_KD?1siMPJ^RhKe#)qhiS zPb-Z4aYna6gV|_)M|s2J_gnxyE#81G@>^g&CNz z(~hH;J=4r!(3p%nd9F?VGku-IOdVf*U!zU(*OmrZYCTtYRYwI1%wyaFqx zM+<1bE?c!3-)T5p?&qp(U(&noIWL^1C3-^PVm6s$zO%@p#BrFO-gc;({d~|9HDL0W zP@oJ*0k$A??wp}<>VIoAJ=9#_0D_&dOpU4DDM$ZC_qGNCBy=onc{s^mk9g7tKRWT4 z?r?<M=-cQ zY}`Y_v6xh`p^)CN*TSc~qlJki(|P#whesgQQ%=pKVb2t%k1NNIW_(EH40Ilr~(E55SGg!|0|Z; zpVwRWNlTM=f?0L%$`u#a_ad}UakhSv=PRl1ef4c1?~+`&8@4On#r5iLxDRGK=@=b% zgdcq3+8JvQSjp|NdX5A18@iwO%tpN&1U=I(`G6>C=^4yo^2;5m_|fTU_x;k*;7;1= zDm=PP(A|?VXy2M_n7S^0e_{zjQETW|6ENPQlqROUk$%bdI>`N^XQ~hk^aXaF;mDU@ zAm5(XYd&=og;)`@tzQePDVHfGu z^}U%gd<5Y`MZ_ARxTyQE`~0_ubL%}DeD(IN$!-xu;EQuHalM3%r-?I|X$75~!En`O zOj`n`3(Z8PhK_1goTjE7=|eoPoZp`9$VezVv&xfKvyx{p+nE4Mo0r&#Xy4r568BKs z`KG1KVkQIyow;rJ_gC!-vniTy9otzHh^RL7Eo=h5HCULwrzC8*L%7qR0B)YrZau)M z*+)F;q~5H8l6Mje0#m?@BUhuQr*PmDfTj5z z`Sn>SDcB=`j#V?FrTJ1oiWndw#QEf5^qv?)^gpwXgOsqM$t%vIRdBOi4R+v+=|Y`F z-8C!S@4O*D$ojrIoz4@`hpgiUC=o7pg3)|3V$%NWD-2~_L-TVB^ zxbM~TLUbX}&G`E+fRN=KEK)yVa7FEkAZPdwe#+o7_~3Oxjs?PwzY8m?LSoxBHVP6i zo!}LR>8T${>)W}X)m0B96hWP=j3RB^pAbSnTmBll!TL8}FLDEVOq9u;#P+01Z)Zjp z+{uLe8T+&Fq>AUs26a{*Pj)dPC2bbt^1k-Tbn`daiF+U+h1YB!pUJ-H=Gt_vvS5ty zTAHmyasbUYQBOhN5#5T}U8w&FBx+74S zCAov$bxThU10!b5WU$1)Q19I@bfc$7$pA3Kq)qg?MO2>>05jdo)IK)oEc^X^-f)^$ z;i7$Rku@f1eYyhU)D8`_gJy5Ie&XD;;)w%%vNSu1M1qOF{2lq4iaNE_fgKDz-lC%C z{4)lCJq6z~zc#XX(JyvgZ#tBgX+N=)YSii~G0OE*KR+yO0)S@S->3Vv5 zX~n(%enSKpbc!4+0U3TenL3gHGrqV%IjP41P?Jy$eSqGB7*zn^>o|jv;uK3l)HvyBIJKgMche7i_QEEl4@)U-4wt1RVuys^DyE44o=%n0U5 zf%7nRQbG1jOr{OM2NQwSDxc|6^dONlYOEG!<;9xMQN@`uB-g z<<#Ge+~38xsdL;-Zh9S+S5=~Lqr&d_3ReI!SN~%6y`4#OOWXT?16)_q8-DI(Q=6Bm zLJof(e&7)~7^TMB?bWF)c~vJS!Vq^!lG*<^SYi*sEhg)k*FQcgK_bue8ImVi#`8iKzQ=LD8Zn$xKK?RE&p34>v2pU`vSR`JiXfa{7Ye_GnbXs z(&atVgtWbfytcpL&9M(O%w}qP>(B31SxpU=o0dHbji(W_wKxlk!r2f2gzT>al0riR zRul8kC)02yks~~&SY0q8BR=V*`^~KP1~rgfvw+uOGECIX()D;ptpEuNTZyz~-Yq7{qmSCdXe2KLTbWnmy;u6+^uH%ftH;HIyv%W=qEG_056?Rewt; zWTO7g3uobH_&*eNZYS_72Od6P(XE@;FLbm7Gun1`cI@nhfT)b| zI$Z6)^>R6g{4VBk+175_o2q`*54U&xKrQOZYrU(&Vbnnd>(Ems;nY0?(dX;euMj94 zDgXV!q=~7ioYYTRyHyVSc5k~TsUwPi*X(vO72T)>5^GHpDyWJEq)%ua86twMX52n_ zz5ejA-f5W&>OJzE&&fbP@As@P95nLx`Ge?!%2*WBNY553IITrDKej&V(($4z0t5N5 zUY#nPSFf)3TJel)dTkR|z~33qcUb_AYwpk$KEpv-AerBwQ7@MY=>KFv(-1JXy)#`S zC%W1dyBS!*;8kTc(=LwS*fMVav59{8*2GlOQNx(!Y}++8Ij{+ugNFV$!TgG4MD6iLumk6n3J`N8qXrU2&)PLH9nt=K zVl9F6pmLm^`v{zDjTPs<0Hboql#@rbc$t0whC_1aHyu2x_M8<%BC-Cos>mufG)dR& zR5!f!Ep%efG;RTb2<3s&!#lhza{Ha<)yY8fvlb5|A7h0Ew>u4&+O#IvU^r|vE6gpq z-u77AmNAG#>1J)XYVPnW1bavG??iX>M+z7os!ylkN_>|_P17|X4hD4mMlHacWY_h# z9Jkt=(HN@Bn^q-VvzsITXR6#XHz@FqrHSt8DF(U&e#yZ)WrN`oa697> z;wyUJY%QgRFp8`((Q^Q*-5y;0eSt%{bdm0PksmpgZ)d|g`qXh)bl;DizH>D8A0!P{ z68K3lT3-v13;$WkmR?^};4_0w$!S`h+Pt?pT}!`2J*4dowpI3Nfqt}a>K?eC;gs`G zd&m2qs2)d^BY|fJR%S_&k(-I_K8PL=G2iktou2cQ{Ism`R@dmhq?j^;9`rs%-yl|x);7o71z->10MaDeaYC<99tSU#*@O7P#sn#r=3NuBkz-Z{npKqUq+sd{w-T*J8v3$ga%CtWWN_17c-ZD8eVpe{mVHQb5=Ep{}ZKcWV|eZ=*;? zrkTx23eQXj=s5n0B!vTbfLJF`>c3ER7h3tPTl!A!0Q2OYbCRNA_!}q>pvt&Gi)ic` z<404BT@yC;AF=iNOLSxCjbVB=W55+xZ;@`!26^-*E|QW zU5$QLG3=}NAGf-+X+o|DvN1VOzftDYBfK~ud9qQu%x+^Ua(T4fJ#_~h-{=zCu!9{d zjop(MlpmvW?p?V>~ zs0hu_KkGOQe&~D?<#$RD^-qb8-2=Y@8eF-O?#^w(Q!RtnFx`4br?syHadB~1DGZnk z`8-Bl3Mss!x2A%p$=sX7Y^pFS6t-&ONAV8hSsJcjIn#^8DjBh5e*>Lr)@v1XBu0@=`gz&-(dm8A9#0wB0&RwI@nY)pEkH2YfDAae|}YF43yNI`t{YIUQE(D zi_B=HMXUeqX>IIp#%Jcq4N*H2ud$bV4JCiFAzpVp;^80I#&me%Mfbdf1mpwUou{{O zNCw3!SQ`2pmM-u%%ZVibbJE~76YtL_h&+g@xtOd73rDcM zd`03lRKB??hVn0s#UppHM>s!A=}DH+-FDqTMvlR0wOXj2pE(lGrd_`FyV;kTii(hs z(0MK!AGUpPaF8VAP%t{KJd#5xY$f#)JCHe}>^bdb^OFJik;|DM)p5$5ZU=*T$v4){ zm*V)@0USI8ys9aPFA_(Vy+?+cCSA1 zLgK$n!!NP$a3fZ?ZzUr2H(gz22T#xJa1ah~{|^NTdYDf>HDG*t9^ag= zmiH+=op$$KfJAYCOcoHVk zmu?sR`V#M*NWA;`f#qCd>S-#bq-&nHxA(?q;p5X6p>Z7P0hH|zzdwRFAjUFP!6X!2 z9Rap{{Gt{5JE&5c$i#J@=#B=4$zLWfKmdao@Ol(tp8eTYFG}v=0f#~Vyy7*~XR93_r%Nt|r5QdV@E?->>|7%~|g1*rvKqA#b1^cg8KVJ+C@!6nC^^243?#oBJ6yY8U^Cx>m+tqWX*d?eSG8k_%GdUIV^n?@*I$a@ z{{8#J#7oP|4(mfXslrZpI=Ehb^*+7vGytanbX|0HfO;uM z`AgJH#dL#_N}wC)6~?o zrx(awf4aptfo;?R^NgGUkfJP<9sJin95e760w5ca(P8B?A6xRsGSW{M%f_h5v1U(f ze4uqfISgnnnA=5w?s~Fq&Nf@h%`cW~HLP7dKik@b2v`J_aNqT89+1}?+J2FIVmZkx zRH>XU2B>uq%otORS9%ibPe_Mo5#}8JkrKA!BwC|I{Z9`R{II?KS~dgGv{Z}%&r@{# z__5bG$dgrGepLPa2vuiESK0gi^TSm<3Ld5@$C~aiipT8{ zv?XmQvf-4^y43AR=xj~wR{OR=Y#?>|jt@iz?*!1zpbPipJ^oO#i0389@1tVi5Kt3= zK~HgU@mZ%=#z&HAJ0q+f!KN34F-Jx$6qGO>g`uj%?A=F-;xjzghJ?GR0k*~N9v+=~ zEVCb^18`SoQ2-i2_@-$%#p3{_#g!En8IPN*SC|S)fRw7(oG5!g*W|_J=jZQ#mwy2y ziXZhll@@QXnw)mi$lZ=NG`QWTDlHYxwkFCN>g(U}B;rHjyF0z|lv2Bh69(~j-?Rn< z&#S1Y=rwAGYZdDTfBHmD{9Si4$&q`d30D+-f#I9QUq$gOi3k9QNM14G{(fC}a_9bys+ z&&tXoXQqV7$;mC9WW?m;=AM8Zg~nlb;oRpB8e`?C(N%Jc-~_{Bwjktu`0!x}Wq4>P zTlmR`g&*McDtV%q)~bpMf62rmKwy0%Umydq>KOk&k^8mCkNQjXDt*yG$ogE9`0nKk z%kw9BB_)w!0$LYWy6g88c-}yFc;)6SqZ=eO6`G&1mF z_vo5>=>4*Mx@ELj2a`uyFbn!JP0W!8%Q}W_PP%ytja%v|J$T`;KTR~tx`8N<)7BbC zzZE{>;@^tUS}K?tMPQb%VTO18X7Sm zOB3_B=t&d39L$ncNPGmT08c7y6O!Ebx@mbPEbJC%i0fx~@(D$Dd#kPKv44YaS%Yu4 z)=yd`Gz{7IK3OrLXBgozhaZ4GWyxS4@1VrH`@^`}`#uhX#^O=;ZalJL?aru|07h$^Vj}-y)JeiP9+Y`I<5|e#ZqIUeV zO4g6Mk`FN~(>lK6Ak^#6KN*8x6@xNZ`h6ab{J|-x_5JrGebIxvf10$xMVxl1Qq6B~ zuI;zJzG?jd0i$MQT>AE#V~Sy^f*%;DUwARQ-rjg{r%;7fRDcJA4>1E^FwmGj`Q5LtSSfm3ASxqD^RQE%K*+RC;z8-29jeqItJV!eP4euji}beC*3cSF5j7E=2^*Mkvqo+%eQ5x zG_O_k&TpGK1v*(Sw{`?B`>puruD8V|W7M1B_8rZx-X_-A*fpq;huFRUedswGfxDV!w^QS^(j&Ucmj{EgzaBYu zZf_#OhiXq?Lc$9Q3d}&f7+F%60OhmU5ith}#58&Si*`kbE37uZ%HMlvZZ2Ju(Dssg z0ASyrHnwEc1E48xk`Ge1VlOGOQRsbSqi!Mgyw_7F+hxbC*R%fGq$d^50sN2!%DKTQ z$qm)#ki~S-+%RuKe<^NeG}xNv0s*5Z-TY7RnonZ!D^l~RRcqv~@Lkg?&(ER)5_OBM z=t#@K%H;xj<#}FtqUW_5mCl2Zzr! zWCc}TFto6coo^HvK)zAd!i#&z%K#jiFy|K(FhoS34YjrTyM`z8TW03i+Sz3lM+AG9 zwuO6_8Y!a4t0$`_qr*;V_gfjiw*+C{OLu}0V~sLMNd#Nrc&fcToDZZWe7Ce1A#PkY zp)me1^w}NQ=G^ek&270+r*@7)-C`D4dcL?6hI)7JAi77c=%E;jJn`b&Z!e*j&t(!5 zb(*)vQBdUB>v~bTK*?$mQd(LX9TQ{t_AM@r$nz(T24n3AuA&xF|i;cF;NPB@sGwK zGn84bSC1yZhgMb-Kr6qL z*f~S|E~5va*lr{ioETVRnlkgAsZO8 z1EYmGiu^`ttX~+4{3H}8l%0lDdz1J=DUtuR86dqE9f)hVmyC>@6C3V5B=rgfl7DI6{jL;%b)xw| zT&wf}6d{DcnVFdk$Ap+@RTR!B$U6{;6|_E+Ml*XK{JPYhyFJH&V!cS@C~+55ppx3!l*dma_W%S1@05rst#eOqVagvo0GvreOblbJ{Ncd?NQr}TxIMQ>}RDk8yA=G}ZY$}zPL z8%pyu5ceTOX717~cRe;M)wG(f56|bqK!a&IP9ej_M!;l+)0p|i(YgXzUfa{0@(~Vu z?JC5V05|{lpuq{4T%Mpg;Su_8%{8U3V~t9c;zNCS%`82G_`hUnzHG72Q28t*smNrW z)A3X}w~^{^no`M*j*gap@giN!gOY?~d2@4feqK^k$Z1FSU{k6{Q#nO2D&JQ76((eR z1QRlqiw|W$1d=hbVo9*4)9vwyr}Q5`e*AN^%ihtEGFgnV5SS$d0txyOJBLTp^Peka z4k5M-OaYYtT_P6zKo;e+tF=MK1E`f1XW6p~@Uj2bZl>GrTh2M!*wCr`EYhh;D28%m zw}Y_R6=x5yW~rv*#z^qrBjN`?%~wq3H|3?=823@le+Ys6?CT>9Z5y$nrKQciXD&() zFIQuPTO@;9K;+3+2EY(cwwtcEt(mY~&JUP}NecY^ya|o^!6`7BFXyFucKy$y3qbkI zd#1O)HSst%)KugD_gU+wBV7N{0-S8mUGF$CVS$AB=bAxW*?I*f>eN1;b=&1W{>PJo zGfhFof&i>Lfzlz}%lb98Z6I~7^Jr`QKj(}hU+z&y{Y+szH;xE6ODb?(H5rNEQa%@F z=Pw3LvmG6*OiVxK=Dfhi0UM=xY>Yj;{8r-s`#JYP%r)|rlIVmX!T>+}|Lrt?!orLY za*j;V@r~+bx7%duE(Hc`cGncO>0K=h0+XYMzv}`C0g_CDA}@>xr)+32#P4Cp{@#R^ zb5^O|gxS&N)|_KI*A0&2ZUB5p@0vzUJ(uTR}tE#$Xb+=yinhp>StHdUqb1U|Bn=N8XwW`JS- zDsgh5u6CJiDtm+{|J{x9LU>tHinONYScY`qC#e0#NPc8wt>V#gMyXyJM+L4eZ*J#68W6^6waLbwi4}(dy)Fn_zng7a~l$#(fFK!N;cC~NqX#A z^02OeGwI?g-^vpklH)YR(=WA!#itK9iXL5Pdnq5)^wNnYFVvL1->^RJaRDEbF5uR= z@T?omH#8 zKV|=Nx5;{>t*RQp9IMBe5OzWN+;coEqq-y!qk_>t*Xv+}bGlL_jCh8W%6*=2^y%-m z7xeMc#rn1G{$WTZ^h`_gYq9JZ@tpdYVqOOu%(v`h4voa_JGNiv+PQ(-X}#t`ksHZSJvu{2ldIhqtk(1m{KkdSKi|v8Lo>hY;6vXZX3YUlL zhw))vCkdt0$X(wIJ1QR!fI>V>CcvikXO>L(+w5|HVaz=PZcn{#WsR#pEVhHINiN9G zf8uh$#PJBg$qTXB8oR)#Qohe&Vc2F*fR%OH2V3YNqM{`ltVo}-MnPy*wr_3cYH$X| zO@_&t|7IMbN~_Zc;iFW1`t*m_&0#ty{-_0QUYF?C0ScwYmbo0S)_#5A_iv=zG(eRT z+!;3U@$nfW$DG87)VVy_20SR%`w3ZsE7LRg3XqomTwOCTx3@_s8BqQLFvP4=#KHi# zcomv$mIlQ=Et9mn!{bV)+%=EY-Q9if!0zzy03ec(5UhLib5b**(F>W;(V0U-3kRjD zHqTsM89jTYVdSEr@ytQPNRh9)AUC(5GB-p+r|5AZdJjua@g2Kg4>>L+o6~%p*XUF&*4sSQ9)5ranY!rl%irzpN)!!ke1e_o(3{m zVck-fZMEa0X|#qH!EUOWnyQ-Gy4tF`C;?L%y6U>>m}4^%4fi(nppkfe{r!b@G|SKQ zX_!ii-+VYHT<~~v$n>z7{D#l)5dZCe}N-$rVV@0UNx_tbCbX6S#lBbv~b0tS2 zGM}rE2rgg%fEfU%UWt8DHB`W#ZE!n1*`5NFUvkKI_@8sX2f6kw;Zghuh1}P;@?F&| z490}y*cK@amX1ANHaiRue9C)$&x0^g6@vfmNlH>uQbRu3F|q; z*k-4E+*$@h^|<2nmu3o)V+xA%iwm@Jlt_*rl#ns`3v-Ok#Lmrf9q2vY52#$q3)**m zQ?pGC^*kDs-TRi+=7*0dD`6@d^6Asfc+-v1h9f=PqUcX#0lB?m$Yez1)4Q}!O184# zoPKG@vK|LA-IKl7A;gx2!qK4p|o3VdM9@ zEqA>M?YAfSw0rIgSdoVZdk-1ku?|n1^7k&Sz(QbbQ2aW`3phq|>Fp7@+2if3?Ued% zmYeY%S0{C==8CbrH6uAdUv*8!fN6_nJ!E5BpR6eK^=)13h`6gQ>;2+zf->BY9`0BE zAQr%cv9U2wtUL~i#MFWiFQxM&Dm)~WwmqJ53izp6Xdpyc>fb|8QV|*89W7X*{(vO- zDMv-#6BPN}o`>)+&5|I`a<@&_Tlbm;c_Y9kdYf~@*J(#Ai1^uW)C15(C^Y3B|MJR^ zmjy*WSz{9k_PPB(ThNfNV(;S-I7!%<9u&>K)LO6P$?F(ci_DS9BDiN%C_>VCR~uF7 z)bR)%i8+JT>*^b*RX~RI9j~&U6LZ{pX>VVic&3~t^2GJ%^L#`(4vKt*ww{{W;`X+v z&2NdI=^`BgSy@?U?74auHHN4GeEO@q^Na1_Qw?qc$h=DND3Q0VFG{KzR|XlGIiXGf#PKlFW)gbm42Wve%G7n^Hok+6nB`*-um-#{s*%=lb0n z*?2*^@H6m+0Q_MQATK-Eo~jCF=Aze*pb#P_6NSoEi#yuwZ&;#dsP!8W- z?|S7q@@H&|dR*9>^#uSZ0ycW>b-Q~Tl@G7+++TQBbjQ0iGZSUu>bS&Q=kd*`J1%j-cYsaUVS{*E4f*11F#^EJ)Yw>~ z1hnEL*Y|+aG9}oVslyPJY&}KcWa8xPQ<&E&L=YHLBW6AJmQodABg`%b-#(?N%*3+K z+zNv<|{xKM}P4Jm6MBzzv0j0J@7-k%~6(JipUd;dOg6 z-x?@A2^>=z?V|)f5s}^5-Wvv%l;Rlu$WGu)ol6f%&Aa0`05KH6%1Z*3vcu>a0HRo6 zW&}+;Qg-iv{orHvujth|Cf2CngnWrbl;S!_u;0HUpXb)#W3|=8@f^k3dM`-uk#myf zx7VN$#<%tQyv>#U7pC5XPjBvSTk>d|m_R7Rz>_KK!+9}Yl&K3iI9pteXDyKw1Q!S0 zT%b8&Y{UcexrvqGY@AHR+4K#Uq@mzTUoYlD(78VAKhf;KLHQh^JUHxe; zPwku#{aO?L2NQ1|5s~TE)>acBzKJ8pt%p8{ef++U*C9^7!IeUv#U9ppxg)G5H@O~< z8x#_fnZwhB&g`&^oaK{jJwG*R1`1Qlxm?ZHx#mO#M%kE0#t}Px_cycWODT1A2^oHR zIDM6Ih!4wCUUnANMD>m)_xuHdDF#L$=x!7o`vig}W_bHtgPRiqgjq}seddFdm6lQ+ z9h{!#aR}XgPaN82*zw6f3s^bWF))MU_2JdRl16z21*Q;LJ!GOJg1w)9e&W|hMD@8! zymSkI=UG^V5<(r<29PJ;k5q64dx0R28~9tdH?c@!3=kfgmElP8d&>jnf}aRSDQdWg#|7{CDrm@bnt-B}r4=LxQ<^`-r^sUl`||@MZ7Gv> zwLlxn4Lq+i)bk{MpM$ngGDK=Qe{A*Z>qp0S7NoALD|g$YOp0*86Dgpil5Nnh(GDKG zE1Lk}MvP*9Z7^$z{sBDY(Hoz{$DgR4L@SJHP+VD zlu;~oKes&u*AS>W)c!mP88$yZk7JC!T}1(K}V+R4GeZy-#8gP;$hzYEE59{9|&6oAjc#Dv!OtHeOO zmH8nKBa!zIczodzOG@GA&!0o}S6II$kiB9@P?43W5c4CuhW9-wZM-Z}8RUppYG#Q? z9%&r^VQePg;se&0ARYF?kHv{@k+>1C!GR@u*9J0X0pQkeP0Uj zIy{P-W(?ZwoE#N3K4fUIenXN1N(0Rrt)!PM9sZAw4zR&QZK(~2N|0Ot9+hBZ9IGx} z#okE8!+h`3R8J+CCbeWWIcHA6*9n_kj%ljIttxl&y% zxG`wmbUoJJ&&9%mZiVZB%4{pGg0D-=*9jsN;Iw`>Hg~LpM#adhbKsk~Y2Da>P!Git z{Q;Wr>KYo&fmpM|8yVua=l#9wu!+>a0TgR4CF%RZ-iqsXO4U%k(`f0JDrXzR3abuo zHD-fv`*79#km|HYmV%9x1Viv zksk}-+z?SS83p!xNINn8HV51ncW)>Z!fp!E>zl!2*b}@a` zO4k1QbD@w@BF`PE&xYiOhldpxIPZVr3u4h5BN@##dgM0%ZKJYk#1OictFv=t;O~cC zSBtdyteK8z@DfQQ_w8A97#XK7oW}L)jN9coOAHRUUJ+jFSrIdrBqT__FHvht%jZCN z2e-)p9MLmP;6^Rxef&5^1XJ zAgg5U`D8Vsf`V7l?R~U7j%gHl4ygtz8dItcC2f2z%`9rnPN|jh8S3bIHOM7_H5{M6 z81to2ORJU@7!@m_#M2y{K@RWR-LixSh#NxQE8xxNKFQ4*L6&#MWM*QTuKvI%BqYSd zv;^Q2kW8~`dZKMllm((Q%*&kvJ0Z4(BNC3z1{4K5ndedN=*&*y&u!wBXbrw6A^(hy?{5@07_3-D;bRd5Vl{g>W@cb^BqUqpP;!tP*#$x2rNjgQtE* zF?QMr2nc9PJ)pci8dlC1{>6(71wH^%K=Ghm(}j|G@iK$%TWY%J@9)S5_{qJwK9xaL z0-C^Q1gVSCrRrD*q%&wNiQ(BFRR(UZFY^h0en<@t4j$P9z2yNyTuR;`5dcJ+2tN)m zXXpAvOaK7OF`+g>42Q|*jKoSHu5?Wc07CS~7RP_eL^XDIJ2RiB3=K4=K}V9{dB$QP z{dKpU2Yk8@xp_Q$Wq6JgyM!A0c2kL0bt^3_=1&Y8X4+#q0-`HtEexwC(MRjgMh)zZ}^Z<`_G{QPul zVt8;cs_iYs9UpL&AUu(_1Xo5SW@>6_Ic<)G>%K`sgAI&v3pELgic)iSbav)T3W1i< z#H3}icl5@CG)moOmCV7S);*Wv6?5{ESL%$2?nqtr#ZMr{qcXKr|RA0|HsY zll5a*6nz4@Mbrt2h~`brj(UA0oaLJh-L5Z=nc(#{^FAN}18uotLsKr&GSF{;=M6Ep znGVT*EbdcLQv+?$Grvf0LP`x8WFu8oRS>m-9M7CZ$IKn@WWOr$AcH1Hb)W$O$l`yh zM?QEqPf5S{EtS;GeuP&fy^Xp#C6p7W=CE(Uz(IJL3PTjME`8}`3xElc9})30JlDTc zlp>JHDw00dT{6!w;p69fi~)gxF%@nZ7n8EL$x z$a8tUhU2f7TNT5kmSV0fenLCZ-9%`*pAxHVSN&d@=Wdncz|kg4Ck&CFK^76T~#hZK^C(3{8Tm12PpZFD~$0y0COF(gjTZIhLl2nqKl z1bvGC<*xGEQ{7O2{$t%0A4=r&;Xv|*g{2qgM&$AbEWy)sPk)1GdtX7=xQCnvS8n?M zaP^g8Rc+zc(j}mzga}A%TIueP?k=T4x?5TrflYUpfOLlf(%sT6-4X(Rb9>IY_dXv# z^bZev?X~85=Xl3E#+Vk5%0xiV6g~6>%?}62`re*T2vr^1TSXJ%Wt#_=@#QrGy4GG=WfrhN-qDo zo$auoevJUB^9;Z3r_D<4KZya063j9pJ7UY#4xjx&wuUYM{M%t;g=G6R{2eM}P4?!l z=jS_HjF85Wj~`rEF3MDHk)&|cit|6my{UGXb#NC+)`EA;oCT89D||!ML5HlikT=}6 z-}hq4WWyvVx+Tyd9bIbyp@jv+nlqC?LQ8@gWD36~#8ZClC z@!TudlmUXBwQIlmK2-g(a3(u4Dk|6;*!*nUi%*;Zps0z6z<4@Ao(k4)@C}76Jy5$jH|~S-QU_m65vpk zZogi~_{kJ^I*cIm#JLfuj4-bz?a_JYe*6+C*Zs@?TTR>ieaaZ&1I`o`=d;J$8iVVO zjdvUit!O$|yt)m-XlM|CcTRYmQij*I2on_ys}sl*Zi1WjPZZKy&?`SAe?bLQkGH?Y z%-P{5lZ*VV?w7CVf04x&%@!imSuQ<2C{&0TlPTd43wm>#U>t@Zv!KA4?t?vmofy38|3j<(!h+o2-9@Tof?W;gsT|w@ zfappa@oysNi3!0+5IdYD2d@gof-7nbT;jL0s=IevXHTkpqhK zjpX|$N^$FsP4C{lbJ`xdIj9-6Ao}KG>&SDc@W0Pk1Oa+-$nRTdyP3CeF-=7g*T31d zNFbVhRT80tp3U#^K%b)MYVh>yHJOkP+J}m{#uo-&L)x;{<)98PnO8bxwbeI=V15|U zs>XNJ`eH>`_v(FvhZublok4dNVzzV2p2R}c$4Ag7PS@8GYGNs2ljDyHMpFjGP~CMd z7OLB&6(>XC!9nzC9S1yi2vqYv`L^`_JFS6Vs=Ne~gdQ+k z2dp+sPFz@qqqf<{SMl)h5Jyz?I|KXY)b-}iH>ES>x>8YM%suW`NBUOoa&nKE zC~VGOdXGx}rvUheI@4-A)KKwT#M zvr2cj>vS7u%X)boY#u-vM2U8oZxrTy&AsHj>qMAWyw-plhkBNrpo>TgKtigGltn+9 zuyl9+57RvvG2s%mQr{`;#Pt2Tz}wEiS=&iXv1`E>OStQf-AsM&rt-Jw6L`5go*5{n zHoUtV%34FJ@)=7sR1`L;lxON-5Ahz<`;|;~FfPD;(< zlbB-VE^%cT1qvqBpc61`^Q;RD47`z`sxyN#1%xaf0xKFS>R5Z*9ku^U0e*g7*Mmmq zU6r#>)P8Zj8$4hG{sz$dC+%3Nu+O2w|EH{zz(K9*LS9WGiOdH93I(P!i43m`eQS8zlMOe}e6wV{6Lq1*Yc_yB(bdaDy2opGFqNQ#>sQ_QInFNxG26uH zrY6aTwlG}mNB=pO_08qx;862QVYOpKDCznODpid5ctidnOa*o!P6MveasKS|c9Z@6 zg8(j`;QX5NJu*i%VLH69k1@LWB+otiViHFqsTat-m!k7dnsZANwd`w(^Vf|m7@_a} zRArO6k%`=3XP|17sxhm!k-yqaiEUd>WhfA>_MtFW1Iyv)KzT`x4H<)L2=Dn1^2HjJ zcI$L^Jr+cTcIzz^kf?#03Q(%|CCyAUHm}m4+;dhxUJ1iaRzLB zSc9O_yK)cY6YZXt^l9a3`h-uO2xq7ZIBve#BO<|oB*;+XFZqJ*0f4~GD-};qUT3(z zyeZ7j(sX#?9d}zShmW0iC#dtx!R-KyB=TjHpRU8f-0Qe|4kD!hjdC3c0hj{ir{vdR ze=Z|1ubvXy%q+IIvbqMcdax~qih<#Ev(8lBRK^St)atkciXE6JLd8y^fnVePX7mG1 zjHibOQCnNN%0IhL55SRN_wfiW=(Z4#;n_E1UXnbItNUG>2q#nSn{47fw1iJ`6?35_ z)BWPL@A&Scr3S5N-%v{KwQ*g(Y}1`-xPg5KFhqot!Qkp2kC~Z*8}N}ruoy=bZu&*| zqWlO$^EQ6Rk^Az#n~;I)#RN;CcBxwF!+uq_%MYu7;^+%4QCrjHZviMhfm*>*h9ipl z^TwXwq_x%P@f3k(l;1_RDoJGPT_o(H23;h(QwRfKe%{jD%-O`f;(ZHP_I0zioZx$7rAW+5Q15GS9>1Jb)c*&>0VmP<@@q{H;8{%~vTI?*r`wXoEOf5jNk%k$6u6@WLu&1tvyMrF)x#YZZG&(18I%Y{Qu6by>Cz`SM6R|NsqcM#y?r??C0 z=KMpv{#XdW4@8Cx(wCx*Ji~EVP@9|(6_%DMQKK$0VZQUgCcS_6jnZG35id;YXNbv2 z$1~#J8t;y1ad8{iRyH}fdlp26(G07*eYqoM?_?r4LA#MtaqfFr;U&tbSDyCWcU6lV zZ^G%wvpmy&c+BRZW>s@3WaE6Tb=%IYCSRj0*jvIV-Fg)L;o_^_)q~!sop>34LXNq5 z0a9YNw94-8?ui~4SvbcJvVM5*wr~}8%Kv)Q1LJ9#^-!_(RiCa_gNCjSu#DipOE7r= zzD@AHUYwjXy6hA1uUg*Vz8N1JWT2qo6H+Wg%H6(f$XhS!;`bw zVlH6cht|;9_1wjgXPtZdOWxp9+S!dZfk5Pu@a4lSbvy-E3N3VH~9_-i}*$j4Y@@Uq?KaE0zdHWiTHsI(#{$(*DKel#=y zs;t|M#ZUFrcC=b2p10;S!5Dp0pfW#KR)3;rCjNIQ_HTzUaqEXemKwIZ*q=sramKx~ z={aTjIc9P$@8p~E)$!}=+3{(Vhl4F&*Mm~s*%z7{&c~dP@B6`9wn(rSnH4$(PzR`d zqF_ZD7#K|9lc0$huJ*^%nuPUZaATFd8}<8kmEy9Y(@D z#Gsmm5v{Ouco5qC9AVV0kK|OT%pm&%dLQ<=`Q7O*R_*6EZ@Pa`| zG@Yp144RMxaeDXEho9}q&%Q}VBW&5< zAZ7%0)@t%FInR<1sFr&e{c;A=7Esfzu2(I6w}(*zGzMox{~^%-<#B3Rfmp&uvi&4& zem31jTqWjZ)V%c7u|rXnAcDF0qPt#%cXfH#U+EI6Qu})mA&7v$J>YY|-)-^v?dt2> z9^7#&_JIQQcfc?O?5H?sKIUrYpq8RzVhVs(1Sri7TiwUk^|7reK(-2SL^s>5w6CtN z0+~r(izdmy{CtY_-XGw^@Q~_x2_8DIsbi6HKXQBd`t|sbm8LvyE8iy}6D+Q^?@=Tn zY17Z}#RJs3Iv!w#Vf6VCPu4$dw=DI;nKpy24rhCo1SSvTP#QP50bK!7eyipyoa8GgzoY1(jS%)4RVI9morOLtkNE) z!&AX&6lL9;?3DT0#nz*-PC^P=k9mXz*^+EqTz~tk+$`9M0MYsOOduTwC(eSoVsaZN z)q1@!gz}}kjUcqxrmt0!UNHxLEr~i=5*mc?BPJ@up(OZNYEVC+Q6ppaYMlN^m23 zjGDQBQT(5q#Bo$KZqdVf$=NY0(3?>ksW1~$=q%8D(8(4p>>pWu_*YN^q~CJ8V42k( zIs=nd3Nv1*IvPx>b|3wyz$2^&hgmUOjh-ahN))C!+lZk4e#JMrmy;|ESwBOXA*!T} zALP+%&n42F8pzK zbmw$8U4!}euA!^3|Dn`RR zrBonzK<~szz-3-F4iosx&4UJ5I*R@5`x4WFE3;0u^c@ zB|C8OIo3+xiBx044`_O*dttk`G&My@=IFHW(5lW-^gTXIi3j_$N^*afCMdqa)gmzc9w!nc zf?k5}>cE_s2)IxZ)6y2g*x(*(2mswl+B%b_#rKNljf1@fMBZKQUvt1@dZ&vh!zo{=B~HJYvWg;TxUpsW?;-x@Ecolf0Eo=T38J>}uY&MDM~Dra z5`ktIIGM4Rome!-T9ky-rpEU8dosp;!DIZHg( z9wKY?+W%axFRJ8}l8Mtp_gY!khBr%B7kU7EDKv5N5JjpIrH+&F0_w_KG>wv!_V*CH z(y0IMvDQC=%-a1JV8;WW{H9kiqrnmO$-_^f^mW^2tov#6uN91AwD`RzibnGXAGl-^ ztT6HFid9+%h4^08H-Ff^EPJ0D91==No`_FON;;XAF z{%*=@=}=w!oS=aK0O8cP;9=tp4E(Dh{&x+tZuqfoZ~GKFJV7^glJQVCfyrejZZ8hp z-TVD4DZpj$rbxt#2erq;*JlnZtIWmNX|2V6TD;)Tl*gP%h#tQ@c1$=$_3Y7 z2Y-CJo)`<7klJNzf0Hv_*7J@!UA1Jw3=cKOn#4zS=h%bQcp;F74_DKpgBw5dHzb20 zw#2@@DfsVVc91A9E`H3&!eo3~pQBV<4?M_!D<0kea_~Pg>aJWnHp~E^HlL1&gfE*G zM}*FxKmj>YGC`I9MXKC;Y=LdgVd$N?rk(k0-Uu%{8cXZ*UeeuN-=p6*ToudZp9WTb z981O!)cGnSJ#dASnSUV_T> z=5PkJJ4eCyCwKck{3G9jxtne}C3f8qc_-d~+ZCxXCOD8~f+XQ2`LVLVu*3KNHDiK$ z#tz-Kd96>Xru+B)GiEPV`w&jzf%~tVf$_LXM_ydGzi-Mocf1F;^s$872*KMyLGvX?Z$H)I`O{X(@T-p?Z;C>4u$c`3FQd^B3M z47PUSD#$N0$0)hzmPju0f3c5fPV#@Rw7ONDPJxE%ZoUqSHEZr?p!k$a(H7>hUF0sL z1M50Zr^&N?+KLih26<;!YS~}>5l&mSDJyl~8h57Qt5kG_L~E)B z7@tHN;_mQ|Xm;2ZHen8AEO~dNJtuIkQOoI`Ddd55qFlhbhu#DIffk<`XLh&nMFf=m z$w*)#jFu>!QYope&1XLy&l?*X1C}aaqXMiq^T)giuJEU-xu0oZ-$_JR8`NZCIA9@R zDpQw4ghtvei(0$OKa{N8HT$!Qq-Uo|peNA0s?Yx-DtFh&m`Kl+<(XkfQ*B>~bH+G! z6ydE_#YVobC+DDDE}k=0%mhK8%T!ux-p)C1#3eYAJ>vh>_a%6ite}QzsKW+m7vD9( zN;af$D*Du}Qxag)R5L`Vj?rqqOhV$BV?xqw=vvqBtK$`^%66aMzP}F}j&N!89-}7? znpknz_I#|4gE`(n7pq(lYWWZ-I$ulQM8B^FyKsa7nm2jl?L%9c(ewlK1VPGm2OGx@=S zGfs4?<&yD`Ku1s5T#0;Jbx)*=wcn|;@2S5b92*2IqF$E4nTKNm_ z1Td{>Q=S1GLE#X2_7Kw+>+lC9NSX$btZl{sD28Nk1sWaD%}Rfs9(A0Rle;7y{@gZX z%mz?&>5n5@Rg`g8vvkUJ<>MHXTIuoh!WkC4C>V~k6VxguaN_MAAK`T?_7g;EIjS~L zN0&9FEM3?P=*RrZ@IL)vOu)$|{~tgExA`v^wB7JiDc6vt!eL?hyv+-S3gqAiaIhxtP=c#16QGt`uxRgX%U z411WRJdyoyLansf1itD_QYArA%lG6kE**zLGn>3H3%A1nw~zHiQn?b%Je^T89#3+Y z&#Bjmr$XFP6z2GVK@kwOpce>pI}ZBA%^~g#g~it@$D6T_FXc5?qtAkidjEfIl4bJ> za0Rx3&+ESbk>pV~X6eOBbfkv(vU@vd z&`Mb!<0`|YQ^(qiFuArPEp(SI+lbxG*HVPX$J+*@Og5AfHva=~0iyUz+Pmr^16F9y zRAz7Xl0iYIaG5sx_qw>e6a+F*IIeIcBWj*wBw|~} zOz-W zb|uxtVh>n8EuB;8_?de~Y#Bi=oKR{jgPFz>03zFZVQP@nB5Y;(T> z7wk#+f8DkkHTN*KI0u{t_i0%3jcF+YVa?fK(A2L~o$`Jr{G+o`=88CWkBWxG>WBuJ za{teiN9xFn5FT9plfc;;IJTE*$dKo~w-=6ABn_{pY1xnOE^`MR3U~@A@l6vwNeKUi z%|r>JIlu!A*hm*ZI4BEC7dJQag2VVgRD5=}aDhrn__+>Cs4?n`u&q8TD=R=*A~^t; z7qEx@BXtHD22|~J=haTgSfq0hUNm40n%#~fS+n#HgeV__Au_D@`tPZ{7e@0rL9wi( zKx-C(7EI^nX*#J}>CEbLMLU+QyeCjZE^qq2t@zj)%HX2wwc@GLc5gk?zS>HgvZzzx z-FBSTzc5*3qMTmK7Jy{sD9LvJc_%^`)eF~P_??ziZ^d+ zGZCG6g>DUX6O(hjug!rRxBX45Hl*=)?zWYv<#pTHZ@N1Cy5{-yt-6Vg9HdFte96T! zUgx!_nuXKvc5w)QJY)383=J3-o+rDU7`b|qDRLo)zw|k&=G?TkeB;Z45Vmx3OVxpky8`IGeD8#&rXh-EY%IN7Z zy*u4x4Q^JZil$Wqvv0m-&w?V z&zBZ|Y6ctSQtebGP?>(=${1lROTe*_tqxLYx>e@=eX`}9RfdE;Ifu_!qE%Y0a~|(b z2&Iow%3Tk0PVAfN#`Ku|ucYOH`&J$Bcdxg>Rk8|`cXV%6&hxc8ARhp-14I;L`Al}p z-ugo>yQP4l!T9`qSwa@%bb+f)={Pl+`j-yDGHR1B?EW$v(ZkbI(EEDqP71Gm9q^0? zKEhkTYfR6;ux4br7U?9M#`LU46m*DeNRXTxs)RD*EVX0)q}S%XF9|E>UJyt|8@T#> zum8PpKk*HBVBVYh_2=rZ-cWk^xwZk~ifI0`pl};sZSalW@p7lejtUXQj^fjY{z{W6 zFL^7CWAH`v2pZg-1U9Pp%wEFq1S&YHQLI0+_NHJ%a+C>9T9?5#81PbMq`KBWz}5I; znUlyS6FR@J(K@=!r@%Hct|KTkrJ z_*|Epee+sd^(bSD_fgc=&mT?yH724$fUen^+iC)u-OmRk^Mw(Mcp_C`N+_cjEbjpB zbTShvT*34ZYIacl#5rU8&h{ppDhprqrGw+8L%@e#G8b2Y?E~aLw-;I7<1U9?UM1EE zNP~OAxCguMZIgY=NJG8rh%@VP$Gyo;C2c@O^AO!){xjPq0Pozg)p-5$s`K?CzLeBkua>Ng z2U$nguuxClC5;Wz`4K-Z@7Q&9isR1U&-C?EKdDtepzcI$x@D?oMbq=Um2WyA7iEhw z#+IZCrQ%exR3?=Fl0M|4?w(iwNRy!-^R(p=&#uAGmDImICMi4L-nVj`61nLiHsb#n zIU@B41X#)+Apt=u3iJukzNP(bI$NR;WX4M3X@3LXAVr4rm3?JN2}S);bad!qlj(a2 zkDgsz+AJZT!qOF{B3o*iI^KKJA7AfLMt*1bB*gZ=b7Hp|ZdtfY>&~Ws&Y=GEoBa1f zuxF(>lAu7rhZXr^A;EtISqBylcXxN)Mz1+wHounzHqk>)!pNpINHs_>P+N`-$-~hW z+mYgF;rMc1M-~Lqv9}ZQawISx&M=0vio4H(pu|S{gR{W_opU%g_nlpF^!=Cl12wbZ zfM;-z>!+)TYwcTq-j+&>$1v28wrato+0Mev@G=X#a`Mdml%*6~kCAZ-n=D^)BLcdU z8}X-QQ(^fcY}Lo!Q-yqpY5A#x{rQ8XS?#~eG$mA{6RNr&I;d-7y2QC7Tn(>!a_Hqat}UAb+9K7ccat^s z1Pd#Z#Ff6f*kR|3-2cD=$EI{dD8Q>~>_4qV(?M^1?srya077cfk?zy{e?Afww7Z~Z zVgfD#*MHimeb7e5%21VQmhS=fKhV#yZWFNxwwRiry@q!*l@8X#ZroOj9w9v%IQ*X>pO$ zdhTtZ+RTn0@Gu0ur{ENGb94JJixl|Yu*Pf{48Va_D>V@T>I&x1^30tr00@=Mr-CV! z7?`0RZ};+Ow;pnI9|#FhuKT(K_5=}(g@t;D=@qDSNmvFL$!4}~FUwQ_O7!R;m9 zUnICt6gqzNc6D3@O|}V)_zEzYoV4y^eDFz4WSL6!3lU#U>HD@W6DEpgy72<5Kaqja zYU=3|0t1MP>JpE=G79q!zq>X=VhQxtV}*cIG|azP7Av18M#$!-WztTkyPz?g;1Axv zT{=`5sB(bU`{3XJWO9+kn9~!(L7W^Nfy}RenOdHaAmVKXH8nLoZW)g-DP`9Bx)I2{ zl7cbUhalDrZ8z~)84^v(d$HuUayc_-ziQIa?ju0!xox@=hk&zY2&$vY#x}rC)!Nog za@$_rJ2#U>?8I-*M3(F>C8*zNR%Qx#>@6>~1S27_rbO~jZK5E@c4$#~UaaQKp5X^} zyZ{$=P@LqXxTE~?!;i*y#*&`Ag7F8imcS8op0Tm9Q`l=j&oV5Ig^5`p*>1lE?*ZIx zPA1&7<(6n5rBp$l+i6>FPub1wN;ZjET}^Gw3NdLA2x~JbSR!I#Xg98}uZ2S!8r&o$ zk-dd*{jE8HadiQvX^bC}3{wGtn+n~pdE-5|1`-n046YRl6YU@FZa~!zQQa6E7+4MQ z^7Q2Fc&jF#!U}$cp_T;I4V#igp9#;Fm~txif*_jrnmZoU8U(5<~qeM!R9#?h<-m`Crxxv^SXyIv%id!pRYuw zo}Hp@`z&MP({*(aVI2=OWr?W#;{|qAWFWx1{IZv}n?&Ltv$o{G!|Lkdtx(cvx{&#A zjgXKzuZLbzhLl=FhRxBz<>wxlzZHyrr=hb%UrVp*&{zH1%~EY^`1nq;Jsh zu&x>e9smQ4+r;2_e~IR3CWw^@cqj)Zp?m&<_Z)@!k|Du%3#~|4beoP@Vjo{Dci3Ia z65f!*_2*B7pM{^4X?~b*zEsd7y~yQgW_qbEPojjj6L55udxLTNi>-<9286KI+bz%} z$V~6KRZ~6L(+4mLRybH07$WWv0cdY(ZcYt5I64~HIq$y)$`W#HAVzC$Y|Jk#Of4Zq z4_*hpxv529t){0RumE9?-$0xzuJmU#$lBh}F*l;&s=ZjM_A3>T$zf8ZRmd9qh6)y2 zN5wAe0nBD`hbq(r5lKwCX4~g)LBk6+4xWHB4+6W@izi*w#3Wz#$J;X>W^tmXSa6!@ z)Keq9zk-S&^uo(!!@)(QDU?Nn-=nALCjW-(5s-JTx#Hn~8pBC|?hyu`OhdT?oB>Dbw%z5bMKVEDf|?tc`S zq_A}8qfq*S()`A2+r^CYu`WR`X7+gH+#j;ut65WKiGd1Po3v%b|AtIn&r zShFyV?ad}RV|TvcYj|}QtVeX2JXw%9esmNC@ouUk5eN`{sk(}2QAmGvCA8;z+GEQ!Tsk!Y#T?u&M z$3iNl?BHFMDJvklxs$T2JHBAX*9gBtG#hk|Z*dv!Xe>$~v`Yvrko@khLx~fK-aXL5 zVio<7Fuld#f-ypMI9B>%L$aiceg!Y$R=#O!t^xVrV4KfnP|PLb=;tsjWCaB0eP+LP z>0EUf+BpaMpokt2G8FtvI*KW*fbe72Q&dDHH9gGF3I9&T;_&?MLJv**;7&F**d#Z) zI-Y8Zt)<;3$6$XYz<2EVD&UnRD9u@yUzNz(oXMRe&M4i8`)6>*W=AojV)_cZKf`q> zNrMqbK9#*c?$AdVaJ)ZioRZLYJQu=1f4s$s0qZs7?h)g}ha!)b+hpZg9oEGuTP^)y z`K`@)Td*@x59Zc(cbmXblYkvojR$2jaVQEMQg{&7(_RyMYN zF{7%+6eI)?`d#?t%O{;kSj+{l%O04+gRpKjd%F030!A%intXT}QJ{rgsI|n^``pnX z#PlxkMVK2C42gr$XzA3C5?wN$mJADccm!-icYO%8@8Yz^m;BRFZO!9ELwQJBnofpt zCvIp0qHRlto{vyLG{xSkF%92=LDS5|g(T&49{aD9V$m?US^IU=OVUBzl>!;uC%QUnbiT80jN8gR_fx62X@yM*dieHvp)4xw9v^MjC)NyuW%N~;q&<70 zz118}UHdP46PgM{jU*Jt`a{1N9POl?%y2i6{r0qaBkj`Fk*dWEYa$62KXhZqJi>TM zsdIvrZLN3m*BX$h2oeJl4S0BXZanel;zC_sfnqHGIRu=)f((w6<)6GdUMf9w_6Kta z*o+WULbszUGJ7;7KI71B@1m^*?R9J{;sA7T6V7~8z~ip*>q=f<$qI3uWR6m+$2~7S zE9=;b(63v}pGk2G0VHOkox?T0zJNy~fFFR3EZFQ<%ZB9c5{8&zvV;b0;U=qT`aRlD zq;?jPuV25OqN1__?xc&0i=ZI4lk}9e| z;~agPMsy+FCkATv#&=9BQIIa5XGZ|iQg2{U&L_%xJIf!Ix%=CW+pgNS&OMyYJJ9CP zxWmqAUdMfcbL(~WG-{^9daG)^5&rdUr(a27-~hoTh+u3ZoZA1j?8Q>ka+|dj81k-8 zM?=(tn!RImX-@(_){7=L){ojDN6K4U zCb#oiGA<2rWeqwuUz}WyiEEKJ&p(MEHt{7z4<9vVN^VF;Ew_lhZK$W{ahM1dHNN5X zd}hSwydU)#T{nMcF!Je5y3|q`{1(Y@eIpJ5Rh-iBef8pPi*)~sYVVe%>8e^jIf4aL zRoOl>UtA82AM`Usu~(JOD7Ing_<~qow&pqxX-1ZwO1WlfHId1U1ht(fGR#Re^VakU z=wDano;{n3ijIj1H2ndOEB928l@(s_oxrX$Z_?oOuJc2CY>b}lJMu^bnMHzzHNP_o zq#un8NzBp(6O*(GGSoQF9^2OGjQaX`TD%>Lc(FC*m@YmsZJ{Fd-B!QNcX!d_mfQ5( z>n60x$&-0M6fYzN(+oKvKyCZ&7r)f4#CvXlp2K=sw3P>EFjfDi zZ06&zb9h$^>OsZ@i!^kCvr5=31SvP`DyuJ6WB8C*J)7@~M93G$=J5j?TiZq_vC4xy z5`-?f%kF#xte+(bbz#+3O{=4GIKC_rY}jq|_Tk?RcwSv7c5nCDZEka(cn|F#4$_Ss z>sN3Ra$t;pS6;cx9}kN$=M^g+3}~RG)(p??+U?S`gGYfj5|nnn!i+?mxBm4rKUt0@ zX)w4wo-va90k@c}p2nH?xf+i1MCZ4;xo1HMU}&P(cW_3@XyQ-PXI%xsqh~hk+qBmv ztq_Kc?_IRVOcEy93x_SNbCN0v<_pJjaq`YgO`+=C$9>C{9+hqBu^t)jGf}c+v)UTh z<6Tv!#Y5%QS*TW-RE>)}c6RytdF7{dvT;`)3;&;hV`rSbC@zDfL6ex3h@Deh;8ru0 zB~SvJ0m*NcCp6B#DnQty^M|q0?94zXcnEmANpNsfbac>pZ*5b0-%#N+lMpBcwAblU zh!1TV+FqZn!EJ%$@-Op@Nf$OmdJEJS{cvzX~-?Rz+{{x}jcb_rC^s-wO? z$0!bnz-btmv&L}L*Emak+x`eBU!-mXlDM!SJD}?eR4J7-@UFD4M`b+d5lHE!gct?y z`V$#X@xHghUodK${X_RY=|Xkdplrvv?eS>N_6)|i>Sr_|c>>qAm zeKyDXqQn`QiYlH$IrVw`yt~MAdOqqNCQ1g@Nhr@zs(1rdAe!Y14Jv3&Jj49&%BWVC zT8_5}?`Hg13(!%ZPL8{c1jX67VOorZ;}*uu{z%VW+q`<#IX7%o_l@eu$>fn{4pG}z z+Tv%$nFf_f{U&+Ex}G#axDEaX$U+F z(C+SHS*XM!|4#EOhi#x@>RY?a|IW9*cubJ&nP5L_MUywnm#L#DTtYI^tH zewwV(VL1q4z@x?^Ef6Q&eDJieGb&#j>FLLgQxWBz!IDWYPcbjPCn1=-RTtP61i*Eu}tY|33Cy2fT# zhlqBHmc&Xd!%yr2l?u(z{k7ef4a^$}TGG$e`bLq|Bg6{W=Jnd&P~wJCttt`+N|;yms5JE$`PkD% z$Z%(O+c=P5L0+g}C5;+uc-~(BOctkXa@|6IR>wQ`qCviJF;b2oO0dN+cWf4oTd!>V z@x>+K_V$*E87CKq53pd@Jc5E8g)lU7&x1Q@Aug&-X3T?0ti+{=p* z6#cWcIm3ep1!@J9mrtBg5Jg7Pc{WZHffccZ<6-qXI7dlRN=nfU*G%&@ zBhw|4QUD(UcwGy0@$Odizh2IHu7ec%Id8Mm_C(Q)=kORmaBk+K4B~q-^}fbJhOkYb zZFuTV9i;jQ6;-wJTC)tNjWu6QB3MktPEcB8nGnJcc>RUVXH_1m|Ne{V z^obBQ5&VbksR#`_S-m<6>%tXvVUHJxqL%9ki*ZK&^P2HT< z5WU*(e=XQEa7u9O-Cte(hNCV{TWBIf%3iE8{+Jd>TQ8_3At`C(#sG2#y{}(I<}Ab3 z4lb|H&S!=YB-r_Hstg;Py^w)a2jSgudtn|sGN`cr%ThwOf?3Ewe{TE>d&+@_LCmJv?9S3yEjJ@;zjn zhM;QRi*8_!RYRPjEyr*miKq_Q{Py{mWurRtsT0`+@%}BnnrUWen&F2Ia{I1Gb5zVl zxrYnM(V+d2(F@k1_?0Y-6-*RpOj~x@M=M-uklUPSf%1gt1i2bMj5zYs|_k+sH9Fj_*XteI?5?XTDlc15|NS0g+|0g zL|Yb@9b4Nh{66wR5xRTz45ydKQ_0us;T!FX58g#Be0(q2cqywt=Zn)N<@u&YT-Dts zqoF__stX=NI_}P74}?GzdyChVZ6g|AN?wl<2$=ZVeBjS9M{dfPj1b(I5$yiQ0uu>3 z=R9Z4xnRR8ZXK7-0|x~e%|B^Ycviv7Ugdu%ys9DRCRCzE->j>yj_-=ws6Qt<_M3k{ z9GuFMv#-rK0~FmwMaCU0&H`8v0b;NfV=4vM#_Gg)yKX3~7M`dJ?)0P032W5}n*0v) z%)F@BFtMPiiMjQ&FdFNUrYCfcT27h_`8IAgb5dUK^)A^no~bEdMao6d$GC=%R}z|p zTS7GL9f&4fx%Nv%t=(iSq&+-zIA?-Fv?+|1BMjfHU?zNywB}7GW~uTV)iYDeDh3fA z|>;gukl14c#@^|dQZqs-Do;>RM*OT^94$-an~g3KK-52 z_cfboQ=RP{6po}T^8+nii^i)pI`i?*TUR^Z+`8blj9E@<$x#8qeEZE-Fze5=M$!P; zx2!2A;TG(~ya2*GA2Ra)ORgVltug3n#jCnlyI6AAzNb1}(eUikU* z=LWVkGr!3xDdAh_*x8+%b=h0Om)d?QRk+KX1d_=HF*5d78J?q`Cb zESZpdf$`w@UWczi{h|9XN)n%4VYL6H+2D}gV-bIV5%ms|CGA|C?x1DEL8P89)Ppm$ zz!{&6Oo3;M$Rd&)9nG?w0~7wKikEeWvs&|X@{lzS?7qperk?{dB3o%FvhbWix0&>` z_PCaR3Ybdm8waen-&qxZd#O?#w?CdJwy3E|6sT7Z6&L{v3`2zBx}KY3>Y z620u>28*?QK3_XHa}xj{hKB*lPe9mc2pvc=dwyWX{_Kr7_!+4gOiw8A1Afj zua>=R{p{eow$U*##uFs-PQkH0A^R&HJI}d_dVrPvLazoHy0x|Slcq|m6?AtX11(Qg z+H=zn&>nMETIDl^g@yk9a07Tx!@4_Gjz#+4z8bT?92NZim@k-m3`k$jgX!Ypc8_6QQcQZw@eS8_LfNZDO zoc>MH`f9q`=gl}z?ujEo=;pfY4SOb>i#}ozneqO*4ENi4qq>zhd2UhkHk{bAD~pvs zi5(Qwkw;{QHZnuA`RzBc>m9ob740rsYq_{AaL1+Ja&a@+$4~aYvcR}0XhS+SJj!#t z#ASIS6&X9M3&{A& zNY)86#F|5#$<`KYp~Gal@a-At>Lg@$ho654KO(%|LrDnye@wk&U|w6-HrhC8Y$uIv z+qR9ywrw1a-@A>}rSKE88F~__Jcb4_A|KFk1{hTni$BJX? z0O>F_N;LUH80oMCFMsyMT zJcbGKblG17p%hmq=M@)$2L7^@oMr-8bYzKBK>QL3N?-cT3 zDP_~|G^3P}>i{864I^4xRAUCDTh481F20=~(Kvpq-w#}tq^m-|%PbxX#SQ1PEET{m zqvLNUsOJxI+qw_h)Tz147`)W-T39C~%B&OqM0kf=!Xl_8VzrQ&FDbmE19NJj@yauj z(X7U@9hY<1MjV1x@umt?UZRa=Fg9Ow9^%q&+&%JP>pf;_X0MAbu&d%fm$01Ev^9d;J<5A`MV;e2_qJ;HN~EmtirP+%ey0$ zC&`xv>!Wfi?j~F!!)Ad-SN#O7ERrSnHiSZSS@@}BI1fV)uZpF<{?$Eg0t1yX(X%dX zSxqNhcVr*NLY6&qZ0-aeHEpX@ku)qeMK=%$x8w#38h}mc?0|b3fMW`UYU1Kzyv+|L zH@faufK|5#aA;u~_&C1l*k^P4`IuN>o$nMQ)uKlW)EGF%o>i&SP-O{X;|>vk$|zPW zXfPNA`Xx;sQMtBfd>hTws1ZvffrNrIQKsVj19_m73m#<10Tg^e!$N`G)#|iN0_TW`CcJVtAIM0NnoWHmlVORz$Hy(Qy*rheg1^TB-pDzQp4WLF=bBS$1B=G$efQ zo6&`gQR=9Za3Rfuw)O)-@vi)$+RL&2aE?lYk8pw4t-qmKDHE9t2JQ2|{}>sptS13? zXDWccT~v#%E?w?IeQwkS0`>uysE7=Z0-RxY0AvSNiV(85x7F3n>=0o@LGTP2nVVC1 z4OfEvZK&aH@;o{@3D!9PR|Nh?&{^*Y+&?kE+E@ZYH$)YtQ`-mT`%m^rl{Zwt@cz5X zy@67~lk2#anjDmu9JrN0*pOCh6LMqU;x}9Q4*!4;*|d%H=TXg8BALhGZRHH27a7ar zZAdwIqZMzUtI8-HG}(Yu$PzYQ_BSo7SFx(|I^6xjK6FNV%(;lI3;ImfFKg=)rl~|LH_?CyVU^N0s}p1p+-qLVZ7fdLIrvMS2LwHeJYIq0+ft#)+{2 z2)IfBVq1z4FH|a5y}Ty`>T+`>V%kOBBPYzPDRy>C)X@>Z$WUP5pg>@we2wH9mWdOF zzeJeSwMiFC*DP8$ao35Cnm8W;$F^UCQVo&bLJ1}}_}>SiODj27HEqpOtDmo-q2Z{agQL#OPt}5_Dyfo^E)K*Q64m$uNw}!ZwnVuTw4qUXqxVOy&<*1Ft z{KU$dMR_i92Q}jux6vwQ=(|;G7`Hr8NEj9Rx7`p?C(gQ>7R=l$J_PaLDnFD@>^Gxxz2K?A+ol{6d_07=8Yuwqa(I7wD`IhMMC&($DcZT;f| z@^c#cv=hk6DSz1sm@V7_avy=@bdMdK%KIgw_4~xC8Ujksqpqx`i=wVeyUDSaZ|wNi z5wyElviC4GGt&}%kmbBx53wSqsU}IqLQ@e-cvQ)MR<#;aga%b}pxTLmdAlo(pbMGX z2Je$H{RxM()?TUMOVcbDI{7{4CDdcf^S;CaNH=*35x7lj*Sq8zw9J8VE>sqN7He}_ zv7!!_k*^Q-iSDbUI-NmrLm$2@hK?k;S|y|B6bph^xoqH;Fz>=$e+x-wm7pTsUNYnM$kp5aP>1P( z2fXBxr6KGc_;Ur;>^9pHq>C9*@NS9IWEpV^gZIi4`*}YH9tS%2MsqlrNnKJL_61UV zlEyh4QlLbBSq6uV!dDF1mn>YUYx4mxEAx6SJOWL~TmXlJRKE%#jyz==`_2+*e)Y?? zZWXvbYl+AcB`bmn@o2hU2tO35lINP{Ybil;4Ro|1K#b%}lc)EnA|Vy6=Z~{0Pf5ZN zDFz1pou8ut1Y`BwHhYBzBJ+`VeGnECfV6|f6%^#9qN5~yxrokfEUmP6oW9>`9sn5t z;E<4Z9w&4JsZpW6mrIf;*t9K;jk+CPuGF6go>haY$P-~YE9~@o{LaUht0(0Z9W@iv z!;Je;k^5otjMnWQG7f??YN~50GH<4@#fbhrs7ED0&W3K67aKb}J5U+~*YI|=9l)?(Ly5*$GB#L#{VBrLsnh&!NhLhvGGu0JbT04iw<_^;e~68XRa>vuJX z3FLWj0*$V~gZUue2uLys8p`4K#X)1cp=MA7&f)raTmZ_bjjLko^Dllet<2VZ1*Xwi z+=SgUq2FVqCZQMHrrFa9;IUb-c! z4$0cEl{q6u3RT-Ux<`(N4@Kv(MMYu}XBEIHHV(=ENvFgj<&96!$aTFM5+8YVMdS69 zr$VllaS=_WQsEKa_v<1AB8a5354y+W<&dbV@FR_+pQ!ebkA^X?eH$qD(X{aLOBJWHS z3_DJ63}Jx9ZbQG-<9`0AJ4fFGXQL zjK6uIKm^pA9;|L$&)*-o8R^sBDL5a<2m~DL_8!M#q+=}qkBD5QF$P=If{x|WSQ!}A zlFJ<9R&o0`^01)E5eup7Hyp39e&XxFNb~dopbb?Rx!E&y+d#ojdiOg<7Q*g^+NRATmK;B0F@}L?g8-jhn9zhK*sXM^UH?X!v|>C9sT^#?S$OKQ1>e{ z30+rbQPS9cQE5p&w%MO`S;H7nTB6WUn%(P{EMh%udlE^LP+<`2a;1tIgEnbw!XDSN zXO}YMQM$DWF!OKVIxjCTQJ+6==E%9;$=$*8%fCeMWd|^n{rDj*BR?`aY+-AuWT^bT zV~sLTB-d~M1wQoN6> zg|Cpb)3;#I(5W8sGFRCe#C+Tq&1BxGqJyj-u4ytt&?k~CEyi~?CD3v%Be164T-TQ; zm|jh5+OcC|Uy^pm4UQ)NfP_(*K6rG2?$R&i|1j*IiT#F*C16RYlee^HwU#C(Chhq9 z1_+Db26-!mg@t+hygA;n44iH|H%~~4p9y%N;>nm(2L9m}Q)u&CiqF_>yuCAPs(li@ z0JgA#ntE#>$^lY5?K*VGXi!W%o8Pg%vXzvFMurw;^_0D0ft!^P3m2PhO*Cdw*OK^( zg3yODb&O~d%(~TPGJgsYsM_>UwROE)8rC$o=E)t)UJXOPcLerULv4q=!{M@D?ZB65 z=0dy}=kEU1R72+_85XlJf^=o6b;P8;%>~ia$&(*)z7W)4y4w9c5Nz}bSk@;SfTM#+ zf6)iB*STB(Jg2KYQ2=TkoOy0O7VK0$^xl&Y~w=T@HL7|PD89w5f z-0|B9<{RUg!TxUNtqo? zLza23J?3#H2vhyvL0~CM!#skkyt;Cx4*GpK8O=Ad=0~;aSCTxsLmkReJRYm#dnz&O zc&Dhi)RfDU`nLiySkh(VSOW41k_>xOr?~~bO)L;R*1va&xL)A@5Iei%nD|^A{>%U9 z-#4=kAdGh72nFp0@}0R=1hKKuT;5jLA^@SKsjZxxnYOmRo4mC#YGSWorFw0Sd?7j} zdT&F3fQbD%0U>_oIS1uIX24JsX+k3oya!!?wwRc>FoozrNSv}Ic%TJyx=~>P9mZJJ z$9IU&;I3e$0el7y-^OpFQ+}0b6-|k+v48dP(vh)N*A{m&GV-;>m!$_!Ld#_&NmVOL zHSmh;H{vrB*!INy3nl_6XQ*{A0AJsj!4jzP3Fdt;i!P$E!^BfLGiX=dcR~vJ6)ILT z7OOi_oKLlbJ@ca!Q`Kd`V*FIEHepU%Ip36#8`oMk!272>A>xE8tQ(dVVkDq z^Ps0gc-@!thar+xncq94xPjOmq`=(;x?SH_)zpY<%qE*XibV0DjJYtwnw2C}j@ZGR zJlK}J5z<;WlBLxf@?lZoza<=n+TB`yesnmZbOf13@7~$zwPyR5llZ}p*RVu*CV{xv zS5I(SLfV1K==0AfX&$$9SXjx^M=lHmN!nE@{aev-aV%JH4a>X~6t0j|IyRxZ>s4ct zI=Q5FTIY>bW05@SqKe*w{1^qw<-n#-T53mS^yd>UI%w z0ACw-Ak8%WtAQ>-_CE@cW`Gvr;o<&Wt;eIH3F!ieS|PIu@R2>wGaK00*wib~P2(Lv zD%ckG)MoW8e>92S>hwfI{pP;AZyPjRGQ6aWEkG3rPyGRJlDuftu#^ znErI@Kn*1{svz-VAq|NEGgw5}6N`(qazZj{o7SFUvXY`vkR$D6^ier5JlU5=`Im}qcaN@IghU3 zASlnshT8jam(Or9!6RKFfnJY7y>#cRoH!T;ndYq_4dEWKWh;yD?2=9L&A4c5+bn*@ zr$%IfyUgM+n_bvPXyjrl^aA+%m7P{CBhw6rpl^lbww)ebE*T6)??#WSJ>5~96sU`% zsLQAHF!Nf8nP_+{3V#9&)!zn<0+I0^Z_{GU+>GF)1$?P|#}c=6Qsh>g`tEa)eg7s} zK*}W>GpE;PV&La>&Ru{fZM@Yb%J@ADZC&1scOkKxx1vGr~X;9YQe-nyM4IlDzT^O&0+m~x3=azU=%;6HqADK z!A|m{#yH4Un1U_ihv{DojGLk!&9l*xJXKdnNf#H`_rf4tiz)R^+Tl2B9 zH)h}YWOnJ?%hGjF0SYP&`x*0^$7vE+$NaGYN^?uQ_MgFP%vcZhmdex|n>7?-S@IRy z&4r1lFbwL2ZBms>*H)9>)`~;IUiAHX;fsb*)}J|S6BMn1qzby~*{sMxJ-GQvx01ZC zdeQ_n-8&(X>>VQJjlLemG!Cea>I6yo{?CmWqa6Cuc_V5mo#gL>A$5&bfUCZ_J>icm z$1bEA3eC8s8n4N-F81cNPWY~QA(baQ=<^jVWUtGVBbn8?TAp&8Aw9&AACcQ$rpS<005WSir0yW2>#6 zOTcw*u8Xjg;m3)8-xetFkN>Nw`ujyeJwN9fwAhu@0u|C0i<9n7%nhe=9Lu$@152ET zAo+|bMRG-jjYXdOfBu<7IzWpCX;twXu$=XF%T@kNpyf}E6< znz*{Q5Nfu~o;kkNBK%8Ys3enbPDY6`gEd`tGkfjNyeK&mYJo9g_eXDs_dPg7Rn8F~ zA{=y?SxhM|9THABWGP}~ekVwJJT5VEw18 z*H>7}fs;{_5g~iCN!l`^AcbXOCUxxwnu`&{rTP)~&ObpVY$T}<0`?lI>19kD+Voc# zN@HafFhYOn{w8WqmKv+xYY>BMA_$yJwxg&zPU>!@FkX0`#%@;#gXl?IU@mj?ln&wu zqHq5WBQgW1U8!%(@|1A2m0mec4v$QSnKH3@f#X@! z1HY0--JnhEk_WjJTnw?n;}?u3)Euev_T7l$jd(^=P;Mlk}IunZssu zKr#{d?;S(-neu`gCgc8Ml|w}n0}z!iujkJlp=@tcluQjn;yMvdgc9l{Q zbg+Kh8C39{aQ@}V4u$AzN91f4ms=UDvLq!DDcWhk7fBnFB=w3r6oS|kb~XNW_|}`l zOG#MB{1scEyIs?s!(R#{ZewmH8722S67Hh&0mAq(*zZzO(wyhxf#K=P z%gs&Tml2o)^e$(%_t@M{^#3Sl_JKx8l;i(?Ar$KPCr>O#!6(j`(~OtB)Wq`iCS3*W zuIyaEevWC?*}+_qA}}x=K!6=uAycytAlZy6&h&P)ElLg~QQC_%VyK?pughpjg}+zV zxz`h)h0rg4wz7nbyFk1Kt|+5(ZR z)!~+}%3y&t38QnlF~x^+&;Uq7TPp9s{f+p0yFzc<_y3d#WB^_;G&Hn@mxRdj4J7=V z<}pj==Vn@x_BMq{1jQh2A@E3VMgElkz7X2HkidnfHcu9(LsP3w}*A{Pq_ zi}r_@L^|y%gIrD!ZO?fMM+*l=dTsz)24?^X)+Z!HuTn3+sB-DTb!w=uF?juL!FYRr zYf^0fQ;Vt#dVe_QGo>%$v|DRQjte>JqScIuU^-4rnB*6nXz>;&!>!9ClsZllx@J04}`rh`hle_EdFKz@z?bqr5+Yu;%;RHlBLy`N;wZEAos~nO3y(+lW{*m~t0&@+u zT7Aia@pQRqOhUOCpw>0?J&Q|UV%HXYDi)3bEwvSE#*sdkVQz1Ccm1p~>m@d20$#5P z;}?b800_(fX#rG6(d&@oiSvOka=KMp3zQGp=CjF642nfyqcIBft)Ozdq4H02-wLb#*bASX?fM5Yjf;=*Df}rV$ zU0kAxqcrs9MRS?9@bLwf=@K1tQTl;6YxszD6B;XIJ=r4C6vrCUnSv-yLwKW`xA{q# zPo|LeGH{xN`>@%%U3}?jWA)2_{9zz-&qTguarcH;6G*kF1W*TlQ*(`+`V~8r;xr(A zV3HB+A}5sS(C3N^Tx^TTB((V;tKmU|BPbY=|<0f;FnshP0)#dhOp^Wh#mX!=3oRg)v*ulKK^ z(etbXsTJ;W*^ze}of@j@2ur1aGK`FCnLV8Wy0eHOw+9IgZFh{%|0)zy`xu9SJpGXv z{qJ+i4?O9od=B@C{pS$aMNo7uZpZZ;Q2-vdHaXa}i-l=k(p)5gpC}-Jzj(QH8ru4v zqhqzsC}n)+bVbY-lv)Apo`V-AlyHIcC<3T+*h zAk2GH9Ult+E*c^$b=I`JX-efJQsTUwu<*&tJcfbf?CFg7-_9qy`T(v^%=_$g0wq|T|RCkG!sAykLYsplterzb~8 zM5;=P)b1!!3zjTgRsJJPvbwmut*DHXh+e*QPA=31>AskhyzBaUY);K@B7-eZC=hDn zBVfYp1%(XW4T#)^VREEY3eAmJSkct*e?6?bO_~qN`V+XWUpNzCm#xiITVLGP)><}f z!o1e*Cz7vdXuO0B-sW?(nwOWic{F2Edy->bR4mjok?va)W$;&ghKrMb6>^_G5U^d(84!b~d?FtF$+$-Zdo?zUkfl;o3*adm??3jR{7IqgV&PU$up86y z8p%LuYu4ghx5kbuMs|GCm%U3rxfx#LhCaoBLbC>rsP4Bir}VlYf)@0yU{vrm$h$5d!5PA@G(}RIgqqDXK|YMLAa>)0_Z!xm|GxZq7n?Y?g{jWs$_JrW#;#== zkEG=Yfv{K5{t#31OT>aFc+f;BQ_u%%#mh&(s$T^+a*o(7;o12tO#h;bcc%_%98jH} z=F%c6Gj~|UF?Z`7jT^e;?U-SztV-d0$p+e_vg%jfKP|ZEoRC{Lm;!E`mHik5uNHb}Bmr+CEYbo%QXX>6zC-VgPP zVr5D|DC688bvd=)9tPmg&)eS~Cu6ho91K2@m8p42Tge!cROT$$RMXQ5+}xD|q%lxU zjzGcfxwl(&srkjIp^2j*piWl0IMB=edRPZ+6vI!Yzwu_X!+9^eQ+nZar&Iv+qswVB!71Do4Bn0kx3lmpZm8*K(tyf@=NfBy9O-OH2bg1|k_ z9z6f$6eHR+ZjETY4RxIlN#JLt$taam!|ziF1K|k-MQ+$8u5p}(Z^hf}&JxVxUL;-} z{p~Nr-b2p6!1X}kHNYnABs$-C{N@8t&NsgEy-!XRvNKS~7K~gz_~AAjEY4_Bj6Q)U zVMI+>2-ljiuhxcV%0{S~;o&87G+tX}c*vavA%7FDX4In1NhJPxFHy`sm-Zt?o`|!k zJ9LqU6kQ6=t82l#0!aQi`ge8$Q0?f`58?0YJbYJmb>Tp3x%2A1tBf2gR5M*P-&<*@7B1{EnwI4W8QKk$0)U8@k2* zPi1+fw4Chd*d%b8!vMYQ&GhCMB3isomoCQTy7-x-5K={*Y<2gyYxHb&9#?a7%oQ_S zY#efA=!=Vuk0C&rvjCi#9rv1#I|hF)pF>4;vl3P6>_NlLI%EV&b6ej0S3tf))NK0F zi|FTYqn;KQ{(`9pN<&$3!vWg|V3_6hNi4Z>Tf`7ahlng?_vbu$vl-?}^RIEIl)C+s zDsnnOj8|yNyS~Oo3|4I?k zvg`WN5T>y~l2}d)7x*r@WcX6s7uM#KIT5egi*0N zIuXzq%4mzN?O3(_vT}dqt|M=yPoBil892g=_a>}6fs>SmXaOE3YGbIw(Q>}^<2ZI! zY#q+3MCxhd7~VEa66JNRG3#;|-ojMo0VVf`qogW2h~c-@>wp`cXkNzc`V&Pt+{;>E zXORcyOeJz*pH9J()+cGkM`b2e-#XvDmoXIsDe{jXQEj14 z_ha;;2=n2in$o~3K4iWE{rCx^8_gA755?=UZwdyotJ_YwE+`|WTlNeT=Ie*AW-`4g zdm(@bZ?aj;9zKSE^HGh@iV)wv)uBSCqM`L$lRAeJzrfn<>&Eqa&&IE}8L-=Ggi{u}x-y|J|5mQk8A5WHkZkYGH zc-lIm%M>=r`qKd<67BzVk`KFSmuWXEW-XqPpaGl=MD%{#8IbDr%cmJXs5?&F&!RT6 zOdFOq#ay>uJ#FZ5TEAoALT$k3kpmWjmt7r@S1xvLuk&$^vW)_Gp;$crZ2&;B`#Y3D zgXJHv@WCnK!C~{Ho7kF475sdUUPFI;8|^sA=%l+;NSBo+M&Z+ESW4{Q@_Ao2&%>+D zO3s;C{>_g+8lvI#z)sBGCTXN-xk+ZsK_>^S!N1*h$7)LMhXdJd*QwPG2|QXiK*(xI zB;sChp@2aiAo^u!@`j}q3&hm#Yx+>h#j?2ueB7+L_KHvJso+6GFNY1+GIDGBuJPo_ z&zKMaKXrd|T^6?X+Df^s#V@SJovwG25DQEu?Oerll2OO5)9tM#5pSE%VEo{F*PDss-($GL=jBI-PN= zb{rl8F0j`$ji!bO;AW#*cXzgYsh9+-_mq5kqsk0@f>p$UgN%a*ohN+1gi4oXM_jl( z6-wtbVp(m6UW+_IA@P%s)jV3+@*}DKf~0|Bq8z$D3>4 zLu!0?PuA`V(Re6qe|JkX^zTpJ&rgj@^IMUT-U1^?Mo>RT{k^`uzU_$u78+>ISMM?C zys{4;a=!~qHm6X_3?gKt+9ydbDA_BVT+Ec`R?uc1M$ zs^5=?SjU`$=OKY^wMc}hQOwfeGnt#Vx$I zfDh3pX&i+n;~{x}5h8$R2tn|BJ$e-gjzU({Z*y~QFS#&y-!outxG`5<>5Mu3d9ohF zX;-Wi%$^8V0*>H)HLuh3et*#71;Auy48YSzE)~#p@j2T2Ntr*1H^@^&aw7_u_A>bXRG=Jj}SAEC76&4u1 zw87EX23h93_=#^R%r#~OrpMEvZQhkG4{WZ9DuZ0{Cy~Qfhr;Yc;mb^w0Bp{V%U-m` zMYtEkN}XLJVcgaGF}hOBa5j>L<8rYe%#LrHcLneKqSLUETd$Qy$VZv$xsS~deJvsI ztfZ&6R#@GyDv141v6SDK|67J`hNb{GhFLbquu%SPJymcNHD4Bdv}`jIh(23E2B7es zMs)FX(%f?}ZFUYMhjf%?3R7Gxn3!@d!OshP)dTIyI#*vW1WTglK=bA=H-Aw8R)}2L z=!UGv!3*YH&?;%GOO8A^sQsYGuNwyPa2_O46AP!DnonzliMpBRTf&t>l}`p!1M}>T zLQ0UqcZY4iFBnJ^w&ml(k$PIjpPmNO$txE61wP>^%2WJrXd*I{#BHa0enmC}c)=rwg#MoBC| z5pOlK@hD>LncmO%4Xt0OUih^86Urzb+DlhGMT7D1HZ<$amI(M%8k$wiCq0kiKjsA9 zR4lMY#`$ES)R+@cZp!Ik^5I2DoCUmts{S5v*Unu@dzCYglAY{oE zA7iD(fh;=)lTwHVtEmf2l_A+^hD~er<5urQg#`@~{fSuC=aS4;t)B2f=Soeorp?Rv zg|u>Z!9SU~I`AH&>CAYPve#xL7Irzfg}`1NxvRoRk}@|bF)P1QPFxlLfr);JGXR&T z|BekD)n8oPHe!PAl_fF>N)v1u@gp*stt=(P2TWfTCcZ7rRV=HsngTN+v^GcS3$=SUY` z^>LW5y`s*b!Xv_yQIKT|cw_9}q{siR)B;}YoHp`~UzMDI5D?2{LxUb+`WA)Hs3?1? ze)83_$3)e=BPLb9kAM6~3K0cz8{C;DTGKSsRX>k`xbHkSS!2k;@l<022EYdP(3MuR zGwutpyPlq=uk8*O5TKNSu6J|qb=ECCAjtH(esjHX{Mg3QZ`ZeU;({=zKcS$yID9`Y zKHgS_94?-)mwOR}wb2DUbY6d86W0dBgygjtSCgtxWT{+nYD1!P09NY?7N?a^95o`X zd7$r(DD0O7q%RulRwY;+*X$22<>X~G5D6o$j&p>=_Er>hZJUFgBd@}naFeRu4E;ed zMk!-KqCvWXdbhd_9`?;wvx+xy?obA{0_XOb$M&Sp1nd(Wr{B#~oh7uiYjZ{yg7<(6 zOUk2HI&n~cp13uQF-2(%0BozPP-)}05^;k+$J}5jakc}}UhiD2d1tJ5^4T2cDrfI4 zKmP%r7Kntwj*gBOpsD(Hoi<IC*`M0wmva$piN;(Z3d<4-BItBli#w zfe#Hp4x#u9b>g1CHN0SvS&}i`Fko}XLopy3fD{`GbCb&HCOS+x&Z>bZwzMwP4PKzP z=vvtI{9eD6GoH!m8^GKy^drD-V+K8k|(2ks5BV zCmzea4*fg0*+vLt^x{43GMGY2z$1ZEid1Uj&9UeNl^$vBl~~U|!LNnuqWmC1Z#|w- z74KBajZU-`qwfy_$lmL9sL67Yr2qdr9-WD&GlJa}z~hS+ENY_>D;JMk4kmr(78fyv zzA$6MRRSiQjKt)Hnf3LRd0j1KNSXUiEBs4Rz(eeQQ%m@{mwa zH99Tl19dgfU%w_@U3K)Nba-sLU=Hp$MH1z!D`X-6E^RI;>8ym)z2hjAM?^&EJzdXB ziFAaaPF}glejEgZQvA^{1EY%@9&_GtK7*sl1P$H{c!E@}gI)%mK5H9k(3tcB5opPq zLlXA!pwe9zUmjH~#Mm%F@5UqB0l+k50*boT?}soCjo|QWKTzi0vNhe}CiT zmPld}lTDtK-Z>iMDS|2fGp^0AVU*_L*Ljg@+%Xo({A2h?V;JRlYA8;d9Gh^ds#!vR zb|3FxDAnO2y3IVADZ{qZNPKYA@&lX0a&7~{lnFGdaVD|T1^c0??NzNNqSMf| zblzoMuY;#I&WZe05U(qmok#EpCvAzx-d*&A_nNzyG)r$ z&2A6}4blsu>ou`HgzXAhfC4p_JgHO1M!8)f_%Y&B@nFdJxcROwT2Pid(yh5v6;n_; zn^6q3yeyNTuN!Ytk3Ap6`wy$1`;4BWsMCwtXodg$96NFD>B%~^^gpya3d!v8q&wSf zvDFf5yw-LT=Xbrx&-i$k!-ISNHQ*KgaWhpQRHs6L{|ls8;$Imbuz%46LS_r+|LJx_ z6>f)FJWVttO288;ocXDquBxr29(zV9Vm>mx+S1c?TLm!V(KU60g9BLbkgSi;a@7-G z>l$WffoYn7i-U;ASqt>wrh|@R>U(Tm9)wHgZXt7t=-#6QWFkDgwAt77+djAHCi|E@;U zs9Bnggq#FqRB#fD0K(q#Xc>CLq?Rb-C@v^^cL0CQ!q4-uld#eXcd`ug06r-gAQ$mA z_v0ydJu?&@m%VJ^xQ(wyLJ)@bnM27|W3>^q3XBJ~YwarwNntvPQcrBBz4U1ER(wyy zA>3J`&KeXgKd5xnZ{>SGsw?w{KNM0lkq|uB$iWMaAH0w5Pz$t0OoVKxJ@+bRa z@XU(k&YJN;!s0PW9t8DRW%SFG{=!=?YuV#r#2ixN{{GxYsmF&D3BAup;@@B1K{f-N zxP2uS*F!Tlbo!wHKa!&}3!wQ+-lVB2DvOZL|H)`xrSAN+UA(@|&sjkd^aTLOo27gr z2@+DWl$CX(w<(@Law5`>ACK7fR*x?jKbM106;L9&k>s5f(e2lPq}f`(Johw*L7vK{cR)ueKC&MQ-?bV8lM&&u11)E&%X*U<9OwDN|7(iZtpMrlti5J{Al33;b+C%>aDeOpPS$;a=ygqyid<6cy#SXYS6i6TlXh* z2y$HzL7{_}aw{?ysgzM}*4ckFVI;=xEchlz=ZuHH(#J@Pl+x4bqxRw*@#`Xm8cr3& zYA2BK#O*tDLAMzV2fSk20M*igVxC;;*d7ydJR=Nex+FLQ9olOkd3R%3YeN?pnY&@J za`hL~2#uaD+U%?@CIO06W_~+o(|YVv!U)oMfaDRE5Lc5XP z%KZXOoH{}>V$!e-+_)C1R$uk~4y#t_Cg?8jVEmtit)Y)+X6JPcG%g&4$1BIj_e;nd zMzS*Fmp3zl8uGile^la?CL&V(J>8BF^%@2**$)I<>>Uq55J|0DVXK|XMo>vVfxxha zDMWAk=T4J<4oZGsyPf;`5BRSjxaL`55O_^y<||Mfzl51Bi_f;tm6D~c#ctfa`D$1m z#+pA&IAZ3 zKacmT_EXtB4}j5z5U4o@qWpj=^eZ6U2{1$eDJYXb(`=uL=X04=tb*USIX~|FVF*od=WnX;V0Bq{vYWIRgHvVBC0SiAVQMxV#9d zQsyRri@my7yo%rRmS*(PLru$5R#a8qEJe6Fz#g^*j@_)mbHL;y{eM~jAd}zvcjXnn zVzpjJb7kwmRt8DJgxN-`8@x-fjr$tGL~`+-l8KY6=iuW~+RWzoRIoT7;FX`F=m~=v zn)w`%%!ZyU=J_-&fJVtg?`LAcxva8nugkIk5TlFsp*`2f)=?|^etQe0e$Olp3|+Uw zGY)@!-eI_QNQ!p8-W?m6HQ|jL@Au0TIgdH+vPVpRyw}Cu%uKdx!he8rpqOKBciFkK zC!0spn<;y?U%NSaAdqw?O`sJtD0uC|R?jnaW_bw-V|)MDA&My#kjA=GHV1bsQomja z<89TP%oN;^D`4H2%!4rzAK14zO$;-grB(0HiTUZPXhBy;23OUKqc^-Q6QhDn1^`%{ z=dq{i+$43;Lm1dQGOfO<6PQF`LOKOzP?Q?>GIoa@{y_o?z1-bn7dwZ`Afsf7a*jnAbN}J_OK-_7Ri6 zjoRU1VYMz!pl?c)G@0VX%es1v-QC@z^~{n4GM{1ZwDqi%uL0db8GQ}X^`kr0R#B~a zU~CN4$OsUaPZSVhrE4#$E<@F{>hSGPYXs+aj>qDe9ZWgbos-^CCa@>?qU<5Qu zE9{C^ph)@2(`#ebK3rZZe)R+KirGZ%-0kM|X)UXrKI?Ow5I$5nau@WxOq@Ds(>awm zTXWCru;1kYL^1F!#kNR_)w)lOJ*!2Y%9qa8At(#4VA4x!oH)M;TTR&;@E~o7&-ho3 z8h9vm*9Joz(^s(H76FW6kBiB4u}rv8~TF}iGhjj8NzP1FBElv^zqHx>8WHdI|5mfQ#4s)bVzzo# zQ%mQGmqt0Gy>txX^s3U=LSKCG_;>`{+|IQ91-2WMfFOX=Zt_Y%sqIKcbd0IwF6rJ$ zSd3~hr!Pas7vt#Rlc8D5Wx(*egU=CI!ot^P^eE40H_!wk)rZ>mNR zC?tNc!EBr-8b9MP+YJR}3yBcoqbKKNXdCDpAk$K#f7|XO4yWOh_mQnsF9czzeedor zix`eW^t(M9lu?g@nE1A{=VDkPUiEaEr~r6*eDf1)y05L|T-km@p=Jehv&$n;95!mR z_Q3P9mFdW+sGxBXbI^EO7$RjsSIWxCr^x-vcXx3?>g$825Bc_6f)+)u+k=Xp+zs#) z87b!4t<2+|>mJSV`A3KH0~b)1IS@1X^1lNm1Fs0``#3`Z@b1O7+Xu}QlX!|K0JkqO zZYtZFus;37txOP1!4{N35}{Yb*n~(^f6|8C^yis}IsOmta<8+dZR{VfUQUSvhcl`1 z5+HTRB2L7ixoOV?O~m|me|Leey7CZdb>9~b&Z>xGW%#KQI6072@@aJUedJ#q zqYOb&YbH{0M3@h?`9SIa7}gJ>H}v9o(Acn>Q@VSGb2cTYEYO`70x^T&Xh85Bvo*w? zEy_~Q&e(Vj;c%wdC%;Xz>!HOCia&j4y`EI83vHR~-3}Ow=MRpzr=Ru!r^fNcfj2wQ z{_H#KUxBIupu7liFKVZfVH}92U%E7N@5M84^Ts!l9zGh}*RLFP7rUkow5Hyd2q`8B zJsnZTQbefbjd+MNB&8b3mev11uD&t4&ZyhEv6IG58rwD++iq;zX45oDHFS$e|*2s$QU_i@4YbBTyxfAM#&dQMXKeOG05cop!iX+%oiCW4v3@z+NWlK z{OLomhX*6Go_#+!WQ~@ge|G+Y;Q`?JUZ0Wb)9o`UVBO^n6*$Yd)&a>h`XqlZ>hB0+ zh}TV+(jox%klj*2RsVOZ8h_hezFrZkN~|2nha$a#&MS?x-s~X;!(kW` zeJ@y#mko)8-jyAbd;k%7CBG*Qh+{w6nXWlwp+Wz6=xq-~OI2chzH#|>iPyHFOuz8) zGI3mp~nIpWC_mQuPWXs7%!!jBd!FaEC2Iz})5@upgW-x%p zET)f_7!AM%ANA}-G@4c@))dXOc{~qO<%-6z8PP89t=KbPV9u*>9?5s$yjE7O;%2Ut zS%4XcN-fSCZ*m86y86+#NHCDw;*MtQJ0UbROTVqn8Wv#CsjUQ8B@!0gaFa@KV9@ zO`;W~7aT*9+>9!=o)Iiv`Rp2hCB#MSrt9!Sj4Xc|3D0bv#_%fPS+B>?>=OL#%HoRUV4|i1+xU^rfPMtSdH_|S#) zlW$#H)L{}oE;MO&=SWe8>jH*~6nOvLJ>An%y3U&@h4rewPy2rG5Q)E`ns~Z?y!vrH zNnUCLHU2Z;a`bY$H>W5>h=zyOP%;eEtlMKu#l=NGLPeQI#=}jk6=Y=oUYvy32g*US zanX&jfa6thrB7g*?E@49UY}<5gkmXhWn=P4dhb9+4l8IoH+45(vHypvS_%!1N;^l%8AZGo66=wedVBY8{(o)%TB&O z_}RZ}&;?2R?&l^A+))53PT8sV!E4_@>Ej;FUT{nt(a>6#NLS>e`3jML?ywG*PHjQ z?vzS0$(V|51m`4OVIE!96Tj-lPl>-;RPm^_6-_AxGd{@-*~NYv>93zyaa`N|cl|yr zR2I+WWEfWLNHZ6}69xcH^7{bKE1yH>o8@wT_j|U1Q1H=G_xl}NfF~sf&x)m% zBjTk>|Lu_{M7+?j9ZO=>U)Q#!Si*MyCu>`|pDI;>FMTxhQ<(x|=CxR&gK@|7LPA$? zQKPOWUe--oo&=K@0Us~hzYLqPeU~OPV^5c_R;aT30zI~>hl)`DbGZcWVm}^ZzxnkC zL;&BGd2O``3d~GWv)ka)K7YoWgxNc*JL7E<;+D$3SOkHb_suMGDz z>g*9aFH}Gp!{RKmz6wRYof?LLG^3RT+12C?cq&O%qB*;+1niG8{TC^Gm8m)fWPF-! zY3~qR3+;3!8d2~7l=kyIbbu1Lh z&E?`Ty(Z{+6>Vx)k9_5|VVGKfEOHNI7g4L0X~8#+I!|$LDJbv_Q>-A_E`i*jZB!_3 zW|rVX?j;!B&Q7C{R&oOpbB*U$O~rgUz~s~lijLV$q= zJ(*baXPw#vK2W@@Vhe(;oq(IwOorP}h`VoW*J9@+{gK3tC1mXs+0+#vY}Kv1|Lky| zNZ|W9qD`^u)$la#3~;DokS^v;P;hI}qB_56dgz-q@Mm5{RSNF3@6t_iE<#|?NN44_ zu#cBSRI-OM3Ph{#;T*<}@4lcE8WFT9ka(FWRjoRYsPev2WQvK7X_K%^mn*u?i;v4C zk?esXI@*2$mIQ$v8cK;_TA49`xxc2X!()paDg636m!JlZS(s{=n_ubYZ4EMHUT^b8 z^0s;Q$Ety@9~CQ9YUAUhqoq5sKA{{ME<~fr5OXQ5su#HIC@5(utsPBBq1IwTjdJ+b zt)FNxt+&O@#iy&Pos^k1v#{BFcqKd*g#e!!JOC`B%7~&x47&4TPZKewzkzx~WGAz5 z(eR#|r@EPOlSh1;X~H&>aB}px{2to3Dk~j&>pV@uam{s+l=M4%C#*VZ-pTk9R^0zg ziY?QXioBfO$hO7Hm?CKr<9jpjT&C)`uI`nVIn~j3#{7VE+WH2U?1knh2Il4Ow9GG) z7>@m`ssMfH`}MP+h&EMoo-Va&>h$$4x!2(Unu2uB>x5qX=5`zoDV!a>z}$}nW!3J0 z?w#5tI*%^>){|(^#;{HI(8TS;2oh!X&`i#I42!ZckDEVD5uC|d_mAw+5;FzmL{zAG zU9Uad)eWhIbGXsVjBzU_UITnx(Ye12=d-?ROM%|*oyid0go^dStKCI!ECG)-?v4tA z7sxT|#jk=f`h3DqnHRo?FIyk)La+ToufXhi4(MLH1#hRnq6ocSG6B6UA)kBT`{nsY zd$#}c@nM$F`5?BzI$*$&`!ynlNZ@%<*P-*{{e?;)$BUJ-4p+crY^IOppZ>sKcUB76!@sIZ^H=wxGRx+LnkW*TZBw-UOY6G*_l## zO(qVm)0;=z5O9hFV;{WsX+v{f)nzuK8EPU4w9zqs5kn$7%&a8&CTiRX)s=q_EEej! zzf*`~DT_NpoDOPLaMt#H))cuxlS5iy{({ptwvGj+ey1?15RM|`PA_z;^y7{GUhHBS z9T%uI;{=pRU{u`+86J-orKSqzH`w_-%!nhcL*mI@XViSdG zEG2rCE6!8D^s6{GCMkaGhYwbrz7l-H)fdJBA!`dp!9Obg~Nbu(8dpt?{w5vjg{%k53jb2$rLa9fsI2DO73I z>G~mCR#kO=@$r7q$&ciJosj)AI=b>)gzHOAs1A;wwk=kOIMo7NvDjRys(Q61M!lYjYwgcIp&_0lD{XG3{y5&EmU_mHqb_ipal zabFThm&+lBDT^{%qP(i2j*)VU==5T0a>TZzdTex@mX5)UBxuu8jr%BQP-1wbKm|ju zh>`**^IrUWfg%zb8oEhmG9wy-THX7jyY+lnXSEGbmm~He&ieqgCrL~3l7~N0nEy6{ z1}l^+eRx5)9hAtc3H~bDDfH|Q;9?Ww%IoLnDRG3JqbG>TaCLun`AoXK%1KWbI;KOc zNMI5~Q3}HdNBvH|MO4-eR$|>uJx;UoJd0nKf@|vhvW(BpX zsW|b)jxKGE_tuB%EM*pz{ut+l{&`w-)8Is*R*16V|C5p~p}IXI=A$2`(Ma;TUw;%9 zcucp3*2_Dmk_L4AzHRw=QB=wJON{lcTMl$v6R1j6GYgmBauD7YG!25@3sQ=ERTTv& zl4LaBrU8Eg(Ozb@TScEoY)Kvhp^S%@_4eA$5z*Iv?sw`bt1S-5#0QJ&S9&mrMBt4B zzj%%kODyRz8l-+@{GNEpoxb0_mgUQ=nG2GYKEv zbhz8y3XOynepdeV?}xFwR`b?M{?5jE4J|mS{WIQ&L-Mcd1iZu)v16USS9vR@ zIu_(G=-m`U1AtT9(HARmeYGc^R$pWfBY)}N-}xyOMh-(b@ER$v?+NoqAr*fq*@=I1k>Z zd%lZNb!bJ7M;p_KHZXY8gEAr`PojO%>(fkr87r7<66U_zCZ_JqKEbaKp)EP7r3<>k zvGn}e{NheRba_alC@vf+zf}hJzTeinnyFwiG$pRH(PZV*IUFjV67p6H{5l{py7_hU zxzkO|c8Z~%3w$sxi!>i7)z0`el-GxV4a`{ZXJhhY-nt&5+H-F`9}RG z)Z?l25hYc{b-`P{fi=XGLgI3fa$g$pp+S6mL0WTd3^Mg}X|W}k~!1u8efgg`RJ>^z&4>Z69$ zgYcoqD<{bm74q-8+?=dw>8Y7<@w_}NVD}X0(tx_9DoxV2Gs0L#j=O^Le|IEW1-KY) zO@Yvf^i_|g#Q}$%k}=_Kd^*CPKtuOIQJVGy+>YB*znSeM2!YS`ZacJp^cXa2ZG+1s zExuk+WqSqOdcmK#H$*eneRW(G6@q1CYKER(u_i)0;B;=IDu~1c=>IccJ-xgML&g`y zuvbjlf+F|AgS54^VJ>6|atkuY5{{<(7~9w8d2dofd76|Wh>Vk~p0*TqLmq~o5);?$ zm6@_i@X2Ti@!_zvMZfvtB9(e=)`Hz{ruDMQs9`iZUZ=O5<)gt%LOhwz<?FiHnq|k&>l?Mcg zLeaJ90s+m62a&RQP{=|U0sC4L{Ls|RWz52n+JfiD8PI{b#jeS&abBLb93@`P*)249Kv`lL0M{0ZYO5$&x+wYB(1nSepPZ5Hz{U8i zs6JRWmcmKfZ&}zd(8Y(m+()O|a<+F>RTZ@j9b2d`KgKjQMbz?73f>3J7L$y0UHBlP{f)IlPLfA+OlazhVcvF__owWSm6y+cK;>eo z)tqlHeV?z&1p9EAxQRkRCi^ZO=@4wAFjF9p1hq>{s z9^kBq_%5>1ywTb(+SYnq2SXAj?^&mb8D%Hb)+EhwQwJu1zC?E0>D9iA;_Vni3+DZ1 zA6Ck1=Z^M&Li1CAfKkvY<7fd*rQ0nqSg2Q zYztgI$J8UEkU1M8{7u3loBvj6 z-IPnUm^F{$4)4KgfcgK*-zL_a(~Ap(suyy|#3`YV*OY934IMQ(QncQa;lICdht6ty zEbQFe_A(tABSPxS9drz^S#T#7oITa4a0?vEt0XL*_zh86Mw)6{})K9$H zp-6KuWYmP-R;D`Uq%Vv%VwzGV-2LX_(_0IYje=(O%vo(}eoW5obF4jqFI%!}8_Uud z4~n`IMlGC|Xf5|CTgjU?l~p8!&=s@a-oPOy$w7JBBlz)iqcK|Ud``8lA6#TB^nBj< zo71T*I5!IQGLcRJnM`z_V@j)P&5r4 z(DX^J-M%=|z>L!#Oxp37Yhq|Jcw)lCn%?HDBh2s2QqZ@>o#2pWS z#uiL6$1&A>OioVQk20T6W`V9nn)HZDRm+>6H+D<{*LG{oD2R%lD}E;6ARN6P-MFZf za_{rCOItXpc~xtK*xv(LDHz7a@-KjGC?j|K+>1uLw75(en;3>%DU6{!PIoV=rBg;Y z_mdLWaQCG+$HXty7u`|#J%yIISTU%3H#Z2DC{s!t$gqFLaK-KPiVR6^*NhvLxzif( zc(p!BB6{%{wYsueI>;T-;WpnK-AM7K9g#jyR}jpbv#y0-T{qxqXb7|iNDN_EMw@dH zer3j^VxDIiWM7J;Kg7O{l)X+rby20|FxSIHnunuxAi~gaz0n(Ukb8P#yxHIc&w0Hb zW7-fzBJ$i0`u_dsqb zR46Rk&zX!S0DeV>{SVj-v=q|CRpbMFE2LFK0FEVK{;%prf4$77&Scxars_ix|0@s@b@>cv)#XInAW17HfR| zLx_ep3rN#+k)axZMQc(8!ypIX&~m+5(wfV@BJT z(D+eIHFA4zqiEr>SVq#kIr_ZGBoi?&HPn4?PA5x+GA^ASMxZ}%%6lCUId~gXOE^!zg9!~ zunb_M#mlTWwG2d{qcu|nhPBw$ZhyY#7|DKkRIrbg_8~W`GQ_Gx0`cWE@QI9>vg1Vy ze__T!Y~Hb?n=bn#)2M?4MnM!i%umf#_K=&5gEW*QS6 ztv!AaS$j1;-|)KL(fbaDoNmXNPMEOwiGz#t-qjs9dI%tc-*U14uH>U)qhp|0pc%ji zh3}0Gi4WKO9=~(S6B8X{u%%?9Awi`)0~@CJbG3bX?milqobY{Metx1l0b4@UG(t4V?c=gP?p6DnA{W#mMCwv*1Y49E zO6FD`Y|Gs5j^kL?=@FrWb{Tby_G8~KZ9(gsQ+OfC(p0Fzxa}s2dvh0y+aTP?q=svI ze-Zp*KRJ5!wCO!S4a7aUiq;y6>JJOp_s$2?TEBF554Mzw1>K6MN4E^>5TbniZpHoo zIeoOFr;mk& zB|-YR_h!|QnVgE;-Q87qSh`X}mLvh#6kl>czP(M2+X4ah3$bGY9}~DWpFp?ncLjw7 zd5?A7%_%aZ9&YZzT%pC6j?p@V=%`k%KamXBg62LWgr!V<1C*F8$tGZfI$hqv-B3k> z(PLX&B&(Yh6HGZ%(T$ztpdpfB5P)u>gm0%_bxtM>ErXT&s`VIG&lQmyN=c%4LGWSS zg^&0fG{$DD%i%DU0(V>7>My$TE=)7jDjO~v0id?WOqXy#@^LNRO*pea5t!%VKG;3S zhgU-WiTWlV3Y@;y^J!G|;?5$17_eWkXYnBB`cEih3Phu&tsW!@2!=s*&ZOzsdGNd5 zrp)8g^F&v~I9EPEMmFn31BaKoG{AVV zy?6(=Dk*MR4YI5w*H9u*{pFzb^o9k5dVrwq_qaP8`*8&Hq)$@;{x=5`ciH2ew@T7| z_m98N5FnY)ca{@B6z-9f4*n3ue4R0R?Qf}jE368vJ4HEIl3-b^X- zJ|1dK=O?{TaL`wj!Oqt?t$E}RpA0}aTB=knm~(1k0`U7~PID;({=|1@p$wRgV>QAM z{k(xv@_#7ij`(Iwd;D5+QK}s9XjFA&?zS(PiHF*GtyKdNB}IbYhq%l#P*mdZ8m`vooaP#MnR#PkT4h~ zPMSLnO~uo^5|57F>$26>QlUcIy9j_%qOsz|Sd83;oa~cwiv8a@vg10z5cGc%CERJ;l{|U0W9bZ2Aku^=nLgH(m3!WBz(BxcF zkRPKDJVMg(Whp1Qz7G^pqOxMQ0T*{Mz4iX)I7N!vrKMh_8r-U}%TGBEo?GADUtike zb`}O_7ljxYtm`eFrtvEQtUp&fbprPGZIk#KEg-I@W~raMMz5(_jLh+!A$QkR@(-YP zzWVqfe2U#ta284C)UPnOo$X!f-{jb|Q;jspgi*vF|I=rgXeF~|gJ?f`z?8y+UPy0K6HHrFwO9{pIIM+h9 zh6YF%I=5!@8U}`=Mv74_DG>8hmlJz_j>!4iE^ul1`thRvcR7(u7&n3SY9o6{=+bcg z4Bc3qXjCoY;&_ho)>Sy z>-;>d%4AwhSdq?b^}@Bl&|>_j+{ojfd=nb-x;1WlKCnb7h3w}cKF0=9^NLnz-aI$Cl6p5~^ehJwPP;G53dGcO=tGxZIOEH6Z*bn!po8X=?jkt=_O zTO~t-XUY^P)+n7$G6d@=CceXkj|dI#@_8k!Z!gug75u!`19ZY~WLxQ4+&gJ z)Vs|W;;Oq1?3_SnOonMmclgF*cR_J-wib8v9=zP~K)S-{&wKls>pX3R4pum-7yhi=So0Bio;p{Gwab z#>U@0!lF&a&;LPfsp8Ol$jVJOQDs5H@vC{2-CNchsWU+5$9;j!YKv@E=GOkhOP`0T z+dcIdD5!L7?9FfGmBt*#rOML1$`l!|%hkTgQlnSNb8D=CE$(FWQKh3l#?u>NqQc`{ za*}v|;$Uh!>hM&OCJhQr`-S;Z|97;zS*@I#H8= zwpvhQ!8fn;Jx!grTjSUcC1`zoI{rF* z2C8@Hg=t7H`3pfrC1~E9@e&i{U9&7+6vM_D)6N7*_$jI>pUXdB0#vnqo_w*8KoQ3j zxBgqJH^ab~QOXCoy9C@A7~VK%Qj0uIe?3l_9X$$Nt1+UCk}u@AlasYSKxMK(;~N^_ z4S}-5q}H3G!=V}#;DA$OWhU!PigLv5KL~t%1^cq$**nj{!q&h}(#H^*5;K zp~v4~a2vDI-Z9&3&I_EagemMb`~-DyZ~#aj#SZIgn6BZ$#XK683y<}FUTFM;(-?I3 zS*Z4(D&Y)Lgki*h*(4XBu zh~J~~Ty`rp8g=~6*IR+HDL)w@y*SCNr+(r#US1;>Ce7{ZCj?bhRifiP zWA^45L(kpilWn;ugeZ2riC5!3`<%`5thT>+S|M-2TJ!-ge-Pr7j@-NOYvKck0djA6 zXm}e^Eeu^qa0sCNfN0Twdwpl8Vf4>M+v(Fj(7eCO2LW|hj?5UX-dsBhL?S+n3WJAK zETp}SGoJS5162CZcTl~wC%oj@KG*cUvXck`%p^FRpzLa^YnJ&aXG5&C|BLxfU2*O@ z`r9LZ;aAUSRUai|Se;(1<1!z}QjYOi_w(tD42o%KjG=k!A{7(oHBImrG_Ajh)@d^o zzWBC(WN(|DrM8_#a8`L~Pcxk_ybrw149j~WFe3$;y<>z8#Gl@}w1VuMgsohPlg>_F zNRpMiF>A$FBu%^IpWr=;XDs#WJRz<;`D-v^htVU&TJ^eU1JO1sHKG_z=dva%00VOgH(F_9$jr6#sFhA5waYBP$VP51;|A4jgYfblFpVt`FO0@TG?2= zZDAYSvX_%tegLY=QyDyDNm6HEO_e8~)~6=;=sI)x-oC zHv>XH#;=dxQbyM+D!d@I(|!&CqD+@E_X^@(diHW^^V#o@RMQH>Ht&xnv7PsS%X8G- zQ^pyjsSa6$kA>F01jdgizuk>KX0LZcc&@;%0hb+-v8);+#L=4rwdKCU8g`etqA;XA2vS$8hUMlWwzUT17jV4GzncaGI}Q{A$71 z-~(qP5vw+Y0+A>rfz`)9NM?YCwgz|*GZ&X+@m!%wDKS%wUlvO(;Ex_M1SA5^^fOL3 z6rA>;x!&EY-!Nv||DDVN?;U=Y`*Jjo4RCw_909!VX7?}GJf7QpetP>r+eNisT1&3< zW&rD4vu0RPW~_K@b90j6l6r42c2fGMvWHA!n{a^P`GOhdw(xd>LQxja9a2fJsy}|s zziHVj>0O9f5fvq|kL^;tmw%sVze3g{&cTLz#pjMxAh@rkr4gEQ=sd!qHt1E)xvXUD z4GdhSH@B^>0q3Y^Td^Y8S;QlVEJ^v9Fp_~}$p5LY@I`=2iRVypTzJI>4D-uMkc zUlg(HUUYSxS>Kz%=Uj)g%T26gHtmO@1`)%nQ{XGe?xCtvN$PW z+@oKB?L01GxEif`sTv+qi?hxK9s&Vii%W+AU#lf24w)7-yxXgVsStHci=l)D3n`f0 zE$(9xINl81MI9Z9S{G$`cBBEepVrW%h zU0(N0?WF%b5fZp^QqXAn?1f7YiMKw6ZpNz3iw0-EWnCK|S#X%`=RTTBpsWcxS*l9M zs-Nh2KC!m8v9aEkx&6K!Aex)gR>#Ug(^}Y?Dn)*TDwduU2ew&v2_Dp}V{HsDpd^@( zAK7u`rLW+wYMny0`Ty#*?m=~D+jK<&Gs6E|gf|emMFim_JWnFgvyAz2+^LYC8o(PjJ^8NX;Cdl#bwODwhplqudpvuL z8ee|2OAdAWxcmLqYJncw;P3~>nGmh6Z6To^t1i+DC1epWdUv=NW+O~m8u0q-87VOB zQ#wi=az7MGG$~zb?#av1OjXXI#fpb>Kh7gnPXTZ8P*6}4raBL@B0`&-PNY$H#pych zRD^08pH%}g1%D*bYpUv0AKQ9pr_F8h0Q}J0@Yq-6qSIw`%myajkW|M46vl2T zK62%iY(eWMmJszD6tlD1(YC3WO(K_qMMe01&;!Ofnc`T~vs1d%WQ z8$_Jlpt5%HlK$qS2%55JS1F`SqqC85$kV`V80)&5Q(~t!p(R7U=(qb)FtYM;k)aE* z(Iv?AdM7T5aB#9um#Rr!F@EbfqN!UutB6q)d?LqO!G2~yzP-oB#?1BoOdv~=G9^D{ zN?tr9El6v@G!SCdmsc-GN5b_J=7&ugie#t$ z0!)^UTq)8b2__$6;P)s&Xd z*Vz-iv-fQPJYAX|3jwi$8UtSI@*0JR*FDhKRBSrJNdW4Ni{?8HVz^Fa*2CvW6v;5~ zpikGTL9#`va$qF?Qy732Fg;@)?rs4R&ajwRnL2IpFmJ|GEv)o1Qu48yj=ee7UvXG? zpzcfiDFK!(B*2rJm6n#A!OF@Ex!VIAF zkDT%A*}C_v%^%m-$E8-Me(u@-P8$(`ZK_xYw5H-o1E0dr#8Dj~|l@ELEs|D&3Pe07iYjReZK zHIv%ay1y&*LG?>ikE;1>#Tsmva*dmoxFk{uw4*JoGIPHIdzVHQDEEZV%L#wS6{*c6 z;i+oFL=33pp4>^;h*I#KN>w>n1sHprMsB{jpKVtdFQ}?N{(H-uTse@L{&8x6aUu;n zbjaY2;}d_^FQ(7HrWADg*P>Y~#AGG%dv=eHBGlO75TZE1%yu*g1xHt!91WJeHwGLG z=j_VMhTaj@@yqXW7>N`n+&~ogz|QYtq?Tn`^qcEj+vW>fr7samH!pLJ(m<3qR|p3K zsbhHZST(MeT*8L*e$v-unVH#Eyp4u3GLUsa0E>;NrwbOEQ&lOVq%y+2l))h)ur75F zuJ<3?zn+bc|2a&TeCHM;PWEy61xyv6qXK|!PAsElts)58QMPip#yaz z8#nzlI7%w=l)?L)GP4`-+;V?q0%a88Ld2K$@Fy~DicIo4{WpK45!6QEqLGso-xS9z zfm+hqz~4+XruPwB12R?IVd?845)pm$@`6l0W|QpcIrrwCKc;^)bRR;=TZ2W6OPMVG zaPYFlCs=fOY2D6W)d2`25Sn~W^-wfzwKf8z{JWWan)`TRqU@}(N; zzBNnEJWfl-qOO;cGo@O^>Zin95zCjy$W2zXNXQ`@Rcee76ry}bH7J>2uuOO5v__1K z12h6;dFL8ava_yQSKc6DLuQ;Rt1cF((D%s;#U3+UAkDk50 zgK51M_EdItWhs=ydRWx7(Dc&At~<7zAdq$5f&a?5592d1UQ1i$g#ZO(Z30Bd&yWW* z95d`td?-HC7oFznkDjBDc~-y z>R-DvWpyf=3TT^_zgpXB_J(pwB&g@I%soBhw#9%J%H6Q?1?1M7z`4Focf|HtGHF5zLUE=|7Mr^SHh)}@G+ zfFRSbrN;i=_m4qm2h5-tf~Z=w$lKhXejxWo-M|KrZ(jkctQy>d1%tCQo64HscK1|Q zR%cmxSR&6=RJA#{*iJCyq7?{ucsNms@jc%DE*$zQ#D>t)0~o;hMs>IQ?c^Q)9S6s@ z`VY#&D_0dmRn@O1x;E)+p{r7{i@f~&1YT790t|#|uyJK8rXV@_Ics)K`e~oS|35<8 z2cq5jJ*w>5ShNQ}F!YO{h!`pdycfju%rE!%yCv(kV#FvNjw!WOgz!luwx4D7=&vBk z$IXpQ<1@rG2AL%-X6TJ3(w@y3D8yr0amXIRd#rCXUV=z-Ca`qvstpsrctFA!t{66G zJY3>Id&Vfl*~eO5DC$(OsfkO&P&Fp&FAb;X(t4p~j@okQR&3WdWX^Gt%2X9xk$bc1 z-T;)ClzqWS1&OE0{Vt07DkjMrFpB@uDAYX(}~BdM}%&x_-%!U)mLGl+%rY^a8F$x5BpH zid9MlAGbrM_kEPY!^1GKu?EE?zIFPV28 z*{*dqqO&YrV)g`kj+~Kwo-ENhT;Azn6Q)fusQWum7>$#@O!*C{IDhfAA!}^ewJ$J~ zA5|V&y%!a7kVsoe>a*bRBAbxhanhlo76kaC+^iyK);0=wIg=K}4y|C!3wfe{HsE>h z=6u#8t=v?Z?`uu^T6S@0u~`4rV069KSGcu5>$Uhd9gCO4c-`JVY<8_85KSgmtp?M-DJJA_ z$k8#-3q^|kxsWi@$*tk}Z-2A@hM>Yj8KSKG7n(j;p3v^hY>>h_sjZ=8z%B^be5|<$ zPA?dYPs^}6LLrG=FM1ANuQ&yavg3M57$bHQabU{2 zijoxyz48)V@dP+nw8Qqm!T;J{P}9*%HCl=fk@o!uxk> z@|bNi!&V=aSwTkqkIUgUuy-8g15Z7G{=?l9J3^(M{L#qM=1$9Av?|!!EFHSmYRW2^ zG}rS=Z+#GA^R@Q{UE}f$=r7RSx!awj4hHjGS^@Y0s5I?`OBar{2z{>j9+J`L-!L_m|T<4xt+cHl){uS0?EE7h`Neep5 z02vkn;m`T0ZpKo~VmY)}TTAIvO0N&25Z~~_ilPRyB8U>}|EnL|ZV=di+Xg%rjLNRs zU);Ap(6aq)ePcr-J}Lk(T1jl>xOFDkd^tlqvb=cCs)XviNkfJ5+Tu3oM$D#~JBbtT zd%YumJ2{rY8Hjn~#ydrTPLoQPlMthm@pJZESZ3eS+WSd5sUc-5M&M3ljf_dW@^lT3 zFC%GVVEEI1S;k8^T0Ekps7XzZYOY=`bKq6rOo38~mM!uv7shvTi8<@%DSNpUsz@qQDLv-$GB;GV_GfwKSMv9iB| zS38rDjf-q$|6aU8@TjBU`;nNNdJ@&9BuPU5#X3v+sr^sNgyY4=lWnBKQc?VP1&giC z&au1%K9BS9vDOHjK2O_6xJ|iCLPD<<9w@J;Z-kljlx(QjioY|%MD`UbG2y~yP~*0f zd=>3R3QH;~AT|3HgeDF$#w$uoX_zd)psFMN{*t}a+8zxW`~f*w|HEbh^ii(7JOl_m~Dy>Y1q0ls5zw=&+5tyMfO7zJi4#?lL=A8Im0JlN`}Knv)ZXs;GxE8(&^)TLES>7tQyT5zJ5(BKoMMqP z#}VE%ZIe=p%Ea^${R6RWjJpq%ik|`*U*4~Ec5>cJb&u(;mLzb{X$B=2z%bii=Su9s zE>kH5`sk#|(Ap`$ix)>M*$^pVLfsP?YRwef1PhRd&sEAHrbv}DuU|1#g9`U&_G?

WL}Eg+-(aiS{^VfKF+ux%6vj*77HsgY?F@MLAqwONVQxuMXxuD!LO90x6Lb$7$UE`V_`uIKCS}a^ zkg_hQ6wcTNt(0tx%R&g2`*@q|dRuO9n`O_7Fas z7Js~izyalHYPvSao2bp0Z9dXkwJq|=@B0sYn3UseRZ8iXh;9-LbZx&^h3GiaPW@`o zJ%&V)$*W+MBqE3=?TJ%Nq@6X+Ub*6Xy1iQUdiNYr!b+G(8o|*0;TQxK zB}Hz`jN=jocn``^Cj5k@51E91fT4%vQLX&L;}OV#3(1Xp`FKn2ao!m?BN>2Z8? zoHbo;`!mwr9mB+g2KJHF*SQzNMIszhpVSe`<%LZV=Wf z`<*J#?<~gUrVLqdAUgfjF_WNLy(aFEOwm?MS!pnScoHCYjOrkQjb1~wEG{od|W6$NpftNZ%;vV87WmP;%nwK~jjDgGNe5%nrjb6n}BBLiN} znl}~$_D14>g*XbE-x(RcfJlPI6PhiBQ6P}aHbgs;bjFAUE09+4(Hlr1n@+S?vI@y+ z7cx!$1Y@~7;g`FJlB?s{A%Kk`WKzF4j8fXTll7-O+DrBl+PrwGfAjdp)N65(NM0Y4 zyO3UOu0jq!rxSZ@QEoV5-nZt_1HEzA8E+jt%He(w@UU_qr&eEI<8Mc&w z;ZI!K=ff<0n1nt0mo>Om?3ei~)-FWZ_Hp3gBAL^w79DK7e;KL@&ZP?Qa6*1?+vHuq# z4K>>?L?grmmhTwb6^*rUTzMd~dr3%Q`L9RT96CmGW+u$*(ODq*R7Mj4{Mu{B^1f66 zv8Y3Q6;|fpaF26-QG0x{)08e=p}4ypW_I>*Su0m~dm~bX9!uyS3(4AUpK~PM(qK`$zjK z8KNhE370o z3k{6!08uH9RxI6r1UFS(E@8OM3kL%{*iGVYa7e5Urt`uCdCD|JeG}o7#kQOSg1`q;zdsx?8$Ix;vynx{+@AZk~A0Iq&uT_riUzwPuYu=9o|6HV!F6 zvAt7&u13~z*EZJR8?GxbhFN0AKgRr3R!zZ!Ro2KOI`pT_YJXfH`ny@0s={u(8Rr6c zo+|XXenM=?=EGb_+l1!Vsbmxx zhG;{s))zSifm;<3-L@ISR-~Gdu^l5LFpZ7<2KQ`6Y&16ytAUOzEwH>2kk>$R_b6lb z4%Cd;-tUpg`~d7P#(p$MOXQF_I3gS8#W@e^Ro`$qsl%qG*LVHfMNjF-W*U+IOEnP_rUr-*S|JC zH&?j9Y}Z)Tl3iCf+r%BxBu35LKho^5VIpntDzR(AqE=ooeodiO)#GuU1~w`qCS`EL z{^&(%!d}P#sYsiA8X*y0o$ZNk-iOK=g8%Ws%$W$JE(N5kt{V|4DMWquuP+Ws(Ykwk zd2L>dbyZbW!6RXEC-8XPz0VkPVdNAP^je#~fKI^Kh5{7T_FULs4nUA`az2$$EV;J|a=kTmbQ~VI9=E78 zGB!fK@%+v&z`?=GR)wG&PzHMxGh}LNa&WL8CIcQE8j=hOnrGycz2Dei`NV1T2O74T z{?noc^fEwlY`7nRgYQhTU&4u(H2#CNt+||>)G%0qoQMFy5(20}#(}$RSCi9{2js^B zn0PK{)Y|my_;S!O%0LW@7VYDPf2FvCKV~$6eYuT|JbhZ1U06Z^2YT*%G({@=4SN@R zQ#O{s=od_gm_4K0>3UCm`o%0jllnP5ivt0A3JiNUYmZkvG+q+=m3+FPH#b+&A3j~J ziz9zZ78#={h+&sk=}uIx*mt9QD2=K%&w1siKoMj&?YqT!oX1VR+6|+b;_C%~fO2C4q_CR=U$;S1+2iB9 zY8~N=@8BC2;q1P>IQZFCKq|F4x%)9K-2pYfc^WHIP{ansDai}kN!K~ zSN8TFvZDfC!HE?7Og&*CQx_C^>tex5L|Tqe(b{&L4$OJgo9^#&4D6huI-F_Oq9sUPfYmx}euW<-2XMM!3JT96++86NDl=k~QT;iL~xz2)#G-yY6eX z%QiN%*^}+i=Tox+S#1GZ?|J|x+G1bei?Y&eRr#L{0UbZH)6mH82{P541uXS7QC%2M zHfJtT?1>27{qM=}H(@g2dlHF|b~Z8K{ebD5jm%7230i3Kux_)tot7MnPdEsP1H_Sk zL8LmC>rtRs(E1~I#f;|N^yOn`3$3cPP}J~j(y;1lSwXGAo7~)KP zOj^UmX~uG6v3~72J1t1B5hj@W97TchJZT9P9msL9$)W(%$-Z$(nH5NS3cee$CF5aD zxrd5TsRwoFH(*TFQ8uFf{>mTUQA78SYh}z7qjlm0*hKdRQOu-EUQwcYQoA?1h09N` zD5G@KFF8x%M~rZ{w>;<(35|7a$Rfbi%LkQn)8p^>*wI#`HTL$5;3lqNObhgS6P!d| z+q>zi)@`(9!nQ6i0|~6mx4m@*x6Bq1+qKV)z|=TUCw-n^1+W^#)^sqHu3#KZPz zZjL~ctOb(bR-OcuC3SVr>Cm3|OZ60?3@a3#m*g!hqaKHL{eR{PR*HyH;_i3F=Qt<{ zJBJieQLflhN~szRVoQgMO;-jB-eZN;+`T|plaUn7;p$6(+~rf_l>nI8ufbhhouzEm zy5{pLAN!tO8FPpGr`glsYkD%F6d=UzTeR2}TRSA>DVEyI6$# zaDxImlu~|iH%m6`_#yI)AUY&aL-(f zNRDxVDdP~n{R=1`1II>RHg4{Qvb>%l<0a33P|Yt4fbIYyOrk(PtAt0R@7a$p@cNKnt|i%Y zHng;~z>SME8CfbTDUlr90%J{TTU1u0e_3%FEX_2D=y`8oM|2_%FdCax4vl05v{u|G z44x86k8+$@I%)W5IgRrBr-|~VY#eiwIfK`1@_JQPdYWSt7%hzB)GZnpj3n}VPj#z1 zj6yzCx%zVsPeOq^M>BU#`<1SXI+r`66{^+f!#_db4sz;~dDU@$q-=`#1~TTY>p>-o zl_zxmb30qLe8R{{fHZF2{Y{_u>+g`&F_x3H6egR{kvFFc`eB-;CMZs|k1blaxeK~& zvgTr~bSHYz*=&J973=wWccp-+7G4mkEb2-Wn&8X!bGwCVauu@2IvBeKwdS`ohg^C5v)*<;=C^K&_nlRuG1U!@^04M80As3&+oQ5a4w% z3=MT;>v_O0zoOGq6aNu+#E9xY8yT<%SIv*0%=B0eC6!wU&ULR@u! zyabfaI}$y`X8%HU6FnKc;IperEMry9{pWsO=`YF|4s0CK>&?X;E(hc@ zkGODuqTa8eEwr4wi5%$2^i34qj)pI<6zn(v0MV=cU48P$#lDOzx*1;fhXIcnw*Y~P z`T_9H!J>qi8wR>I3Tths=E5i$4M@yB&Nu>I=RYA>wFDFNMOdMiFB8ESIH0_LEEJfG zJvh)Rzu(O1e-8yDYbVFoqGf5TDd^0!3nT-Mg9VnC#rRhe6Ti$D79{VCc>=s9QFVKBGaw=z9ZVPz&m*y2FnQ!aSYa67hXd^ztn{r zR@7;ApQ)rL!K}YiGgAJm;_!xrp1*Snw+6&fT;%9SsfS5!w0@olS}kAKkFpr%A0Txb z1`bpW-xhX-^}`jd)Qc`g*#Htd^vw!eT&FkPsE7*LlEmAN+TSIL-+7aA$?RBsm7R%4 z{~*C6Q&OMVT6;ls1@sS>vZV_GgaF1y7iQk-U>=by+9L_%3!r!f$}x9LXfxOFX78(n z?axg>^BIQ3*W1&v_SBdcPwLk!nS$!0{76OT&|SHcqRh(vqKpHLbJ}{qNb;Rt{sjgs z4*FKLxBL7Y)nR~v+O}Uht9k5PGN^5AtOLv&zxCq*Dfol2p+Qcf2thbO5A}LYQc~PN z+9nZCg1wpek83aRaLFq30cAeEp9@Q#iS&cGYRtN^6gH_t0wAyzU|lxOS9$07i1&B# zs4rYu{A3&t?jqK<#zauw^S(FKImp=NiUp{SEL||hYZhs+c{qWN-;TWP?d=C>n-sR2LQcl`$TmxbEXn61qCppY{L6pL1h&i)5Pa?4D1_s`0Svi@+ll|yV`r0&aZ#SV+X(=yWe2IqDRn_RFci-2U&Gs`EwJCBYog{57HH-p0d2RW4LhP3B z2`EyltHtSSx!dhDe8(&*tCEKK>TUdghTwkAMX0C*U=_rO^z;P5y=ZL@Cze3XvNqQ% z5?48@vf5%OSlf!^nKx}h#&CAP6Ri%~KiC@?8)fPrNlpL8_U2qIJJHy}@u5;@D5w(& ztHp|qsD9cK_uuq_1eV6t?;|k%(IpaAMgVbkc0OKhU;V6SC)St4GQUt-%Gifm0fv;`Q`O~weG}tuM3S1Q;T=EM!RFYF zzpSv^^3`BCQH=9*b`brqnxvhMA{39Ko(>{J)T>wYLzuJcBERA=?6T!eRP~c@Stu`l z1(hY}E)3j19i|~^rw$}PaSd$S#I;U0Q(up5IW)VZE*{d3 zqh{BFfL>0|PT$<#G#m@P#D4iTR6LlUMH|GzG=)o-{b0>Fn~YQNZrakuti^o~JF2hQ zdkajy&2FK*G4)nqrMr7AISNo1Eh#RX7G%V#9wjv{FDh@_J@y{Hv~vd+XZp{q&u7ZY zZ)b0hCg&(;`B>YsKpPvULl)ilV;&FAT$c*>6 z`C|t)uv1ryn2=B!^Y&=5{`yxelLib3!$KJz+Jq|L`Pmga<1L3(`li5>b)s9e`3bT{ zoYJ`c0?X5M+C$Q`M7c2iu`2{yI(G)6W#;Wvr|!>kfy4+Ustsa^rJlq7 zuKi=?Nt*}!g~MSWe&G}_Q2q0|9ystVZ4~yvz(yYu=A=Q-X*8uMt$Y$hOUHlU4Ua(d z&{lqK?ovDcuG8hhsP!TP14ABmNa@?$gW8XU-#Q;W6}YaqN9iJ}z6Sr=En$5BTQIR} zua+cDb?M!_=gE@6{?@5$B8xVlNPHY$604@8^YM~@e?CpCUR3l0pt#A}^w9=Bj*RfM ze-ONA4GQ*ryaXmk0R}%h(VKpR7rwCxu#l9B3pCJ2iPPCkd=}kk(Hr0&5V0-J%}vkF zt`F~;`U5<0`I#>;5TfZv7|*SQ&z&1Ywde&?CH;~qDH4A+0(dj}&pgKH4X2114sLfl z3cmX}(A2o@P1FRGZdZG|oxyNxR-7DHEm9O%@(EOz5Zy?&I8+2c@jiYY+;13*-9Hg5 zL;hMUJ8Eg{oNUE)XJLc$>hkjhlGfz0Smw$LUu+`;*i3}fdL}$bor^44DTs-<=o7_> zmM+)yD`6E1{~chWAzd((-)H|~!asO2R(YWrZo=%m=f`VZJ+)wY(@_g6Gw$#~Y3T^B zTd*jK>=Al@+yFk8w8wJIE91t4jWErjZ@b6a38`_52mErLl=1QLWTsQx`;P80Ce%9V>-74)W0ve_YBuk>VxbQS`Kq_{wlb+KMU49! z*;s%zQ}w_Me)~ppaWk>BRLV>Y4W`!kDQC>J}9@keRsjMuzSq`V& zq9cKs=;lR5iTunpwY6Ec`MaiLon2%k%RbR#I}929jQQK&d2DfT9v_dNRr!Jv)r?%l z+%%HK7!}ppxHM6asiBWWRjta0b07cy52(X`upxj z!%Qc(^3GWJ+72 zC+P{sB3DhGT3I3U3XSEV{2)(`;5(Vg>GE1twU5uGqCfNFn(O?otxomYO4wO7tXp4; z!`li4l6FSNALv{aq-R9UiITgI(RUx0x&Y85nNnS^08AA>NhAt2vIELg@Go# z`O6{Dzz<$wN(Hyk%V+R&SgD^hQ!R`nO+bLo11C9eN=(AG!wzPzTlkIVpYz?_E+j0S zM#rLt!mmTW*7g$UcRsO&80Y2bs#Rn;C~gz0l?Nd%N4cHGK4ix#R6@brB%?XOzh+mp zun69*mB8v@kZ_L=4(<V5o7XtS`XCA8Q8l7w(bVwE4(<&JD z-O{haxEi+PwNJ+f=n@jz>IyY+BkLH&zGok;TQVNN3BBs=JjL4lpt2|Ye88?Nf=Pawl%Qp@Mzcs zYIrIj^i-srkFLZ~VEcQm12Ev_1_RNL;>H#1WDmT7|e)R^HRvZVmX|hdDo**II-QLvTV3mSI6}>w`w-|yj zi1Y0y-h;pKyH#D6X?^$-UHU=LtE#2*y6$1e5lHj_|D7Faw}@KH?#sr-cVAu~wVK8^ zmLX8G_w~4J3Mw6Mcw;56I!iMq6Xv7--h91#B1a;En9q4(x`_CvEpi*Vwwjjd*G%DU zDi~-Dy6>c?I4NXdZZ6tdADWRtPE36N`&ZZQ?k*Ji8Z%gwOgzJ$nVH$EQ8Nq(sQwF{ zurxR>_exTq`_Lu3bP?0%)OGt>3A7!5n;ak1(fBZi^DhDScIu*7O^T3DEiB;m0)3l72}7c@TG+!{)t zX$kq{o3?Ea3Ci#7<>Rs~TFk{h5u?j+S{=6&dN+PFo!tLh@?-o*ZD#!7uHGHCiq+m8 zp$78djZ=9t9_NV>_}1N}--`5Qpo%0xucqygjNK22SY(I4wu5#Qv)eypjXOPk*&i%+ zK?%8rgEJBc)TF`=Nl8f=SG=y?vDH1dD7J+lDc!FeC4B)K^K%x|Wmh$pr;ly@;@mBv za(MkuYLSfzD&wP-vY5Nl)u^{w-WO$}u!Mmk*4>`5UL?gu&pgnI1{)N5;GPB3RjR1Y zP`D;e+24OAv{z-7p-uqjvN7vb?P)L^2Uy}Ul*od!z5$fo$+)xoNpCdZlnk{!<(=g#aF|#RH|!gM zsY-(m%Ad?Sm@O_9+1YU9o$p@N(%?-O4gE?xfafTtA|^?LMkJ8vgr%l+B2 zC)E{<)qZdgJR}PSyT8P~W2QVE6jzu=tdhpAq<$mtI2)1bVzVkQCv9zRlfWgBsJ_YT z#D}MW#-9QNufKe!G7MA-4Ue(Ezkk;$yn6l+=dJuj8S38wOcn@Whigxz`T5eB*E`4$ zo>Z@;r%=^@b`IJK6-y1M|h2 zFX;7pV9PF96i)Ezv#S^|i<{%WF|;#w>-h(Lkx(@Fifl1h`{2muy1Z29ih}VlDDmg~ zl&LzYVvk}Tz|_ZI7Jxgs+cDXxsjCAweW!b17X%*Y^VL%b3PTdyxAXiI`?wGVu=}ktOOZZ)rQK(Br|vrBQ{S1rLonc51=66F{n7D5 zs_(ftNF4+`&-Xc)6Sb7Iu^%^!3i<-SkB`sJss_$%QG~(?OAkGtbM;@L2dMqr9nGz9 z8?;AWg|516Az^;+!UuO6S}5G(2KA_@O`mgnO7~^0$bc#~eA+49GrPpg3KY_77qQcH zI)iptF#bPZ5TinGTw90uVrgoqRFq9U0x)rFb7=+nR?lDAi^k?;=Q3>%{^Dx!wCYew^4Si{=PZM0ajS(bE3q(y16kuhW$z~kzFd*v zs0oJ3ngvL{8~I=e*bg>fe>C-UCEeXfi?P!C-0}bC8{mXdIBlQKg`U_YC~0VDYLK8B zVEXvpzBUw0^vQb_v0HRW+7FS!(Xr6in>!~C2hLW*mUv;cKC5klFoQ8~Bogovf%ea1 z9Zs&^M^#n0zis%R?|P9qV01JS;pb}_AE}komcGH&ztKMq{z^ApvD*+lyV1UuZFuuxZwQ*d zvEnA9vtfGdYSMhlde=P6aoj(0*f5Oe#nQ4j_P?ImvwEk~BnFHG(8C5T*Bk`;e-BEuFP2Dx-^$7G2IQ4_H`jpH};3aRd z%^CQjI^@;x55wg8ZLCk$W5W@s_5w2v>paDQ4X|;PHf*6lTTZrWt2v+5dST)V~C z7dNyqbzD_qTTTxTs5a(I?TvDbL4B;G;y^`j3;4&EUlV{zeq_<-esT33DhdipmO$pP z!tP#G#0X|nLH%#;;rwXBN^TAsj&~G4#`>=1tQez)jCOcmIM;-3Zi#vHY6zr0?qn?glPD zEHBm#taQv|)3ytUFJJ=8!`?=Bn{tx1$9V`AW@bnJT)`#i`>jT2>nkQtzr`sZ@hJNb z*aP>)4CNW=5pZ7O4hINTZL%K1$rUn{phs)tvvx6k{ z>-8ci>aC)aq&61Tq;dRpL**x#P#ItOl(urc_DIRC?vOY$FK(F3Tr=Q|qD`KiR)qA} zxU#f4hV7p(-6D~R$r^F33xj_;xejusP#2Q0=lD-U_yR|)skYvd(7t%Y4K)S?^Rg-| zY{XUgrAte!*Ih{2R^t6ul>h5jw>r+FhJU~$ZPc? zAMY;9h`CN~k&JTlDi;^t_f?8W0_aA*TypW_-Enn+LYNpm*{Vo>K9D4BDDrrW@@tY~ z!hq;_>2i`dpJ2bpYC16{|3ajI0q4MiFQlkm56ItcV%D4(ALmZ0fa|Tv_N@7f)Tv|! zOkcDX+MG1*5(|l(BE$lb&-4KmE}KE7*o`Eepw8>P@^hsEWXH$|*)-2(-qmTGDJ$x; zLpp*L*I@gQgjJjqFuc@Te~O|&E;Q?ON1e__7caG-Ug~1IO8VL9^7_)!+@`mrrIwD8 zmD1p1ccOK208qZS(Qb#ub4d-GE!PQ?+{)_e`}^Q`FIvD04@d}X=7f$gaGCG7(Fe2X zkGX^-%y?X?h#?D25qkD4boCz@V-N)@EX;7%;gW+%In`7&yD*|kc)YltM67CssrR^Z zIgz>hFaGj@WEVRSV-UftM4RD@_iNxS2gW~DWbJ|@_Zoc-kS`Eq< zdkp46pbyM@y)%}71l!E%iWu?2LcHI+skb{R1F1N5rZ(6X_^`YF){nne%Y>RocbhNo z7<)=lP||0+Q)e#*JH~*JJ^Gg8;+yg+qw6<_&%NTU|6EE4buPy;&s9v+Bwb%EfQF6gvybGng6nikpA)aHFzoY$C#Lwdy@ z*0=HE7Re0$&%0eQ3k$1okMr$|khniCo{9rR1ceDq6=lUMZPtrPte<{c*xasG1f!D+ zF7+Meq#S%}jKE z!A!h^0U<5uI4HfRv?he9 z=;>*dg-?TXhBx|(iW9vU#00%u0hFG@VU(gkWxBn&OYmpyixf2#E4llT*SV^@MX||M zaL!U@!REKp!!R;3u9lXw^Rqc?D|=uqRSkJOARQzV8uo@Ph@8*U2*`u^>P{Mfc%-DH zBqI}6ljH0v%!QvhP?i!cSw%*M_sBB>YfafqOK{=LnQ zQKcpgT6>rvq+vVadTQ6cyp~dIRVCe_w70nRWbp;~b)ntE?f0E=3O@gJXC7mz?JwbD z0K(KbnX&|~DAswEgO#<_bXlxPK;%l3m)1Bgx+HykU8Op}1$&K*uI}KYpEh@9o z(=$2>x%}y^wQQW{vEF2k3u;bJXOj}A5VZi{3RnKS7XknL$oPoPTQs~$6A_B&fxXD~ z4j}?q2oVNk_I*guSi6vZP^vs-3!n7?r%|1cqdi)+3s>fVH_QJNUkJX~R#)<)jJJCV zKB37{A3x$A85mOoTA$8Zi`jufj!b*UIysV2kx^lvC~{7$Pn-?MoLUBQW$8+q%&-{VNYlnGUtSV(+hry= z#wLq3>UY4!NU5l49h~g^Tw%Y%a97mTbpEyE7Ui%5s0DLdai+7Om|^EeEG;jeN6>iH&|4+h#BzTW!yp;x$%!3!n!lh5alHUg9Ym>Pm- z-@Vukn-|bi+_V|OXk(MTj2Ar+k`}9U;MGhCLT(kd#a$SI-D8A-uKm%skH78?!7;~T z_#{Oie|GyIF2p#QF3$!K`$D6_nmvDg(6zBqyC61~b8>Q=t_E=GPCwB>85u$liZZQ8 zlf~KQfGf58J>uk}I*hP6YPr?x@;{$RtwF%a1$>~&j$l6qavdBKT!f?h2*~+ut{<@7bIt!s%k@FIYxh?v|GMWCOA< z8%w*Uduv(8&Wy_A@kd*r4#(&j0KP^_RH=U2();rwE+#x2ec{c}S}ZsZSdsP? z!32QA?#fTW*W^YwpOio46g2mpe(GbJoC^FviGz!jFY9-g2{g?35uZ~crKjglpkgjG z`|@>#Vtn~WumrIqVD@$0T5TA{Ir^i=JF4tUj_^?SQ3bt63XY5y3R($op6|ZC)pMkY zJbV}vGk@7yVEXK#(Q{gwTKY{8WxY%HBoqZfi7d)_YF9=@Pp!d8X7YEIbx26fVs`2U z9Eir+nr@>n))BjL`ZC%(6_w2oNYU;?2Gmz@Xk{}6s?Zq_Df07jBMAhx)r5Imf3Fd; zn{>BFdfnYrUSAStb#(NIi&MpX)!34|9~3qz{oE~&R-*bO^~eqjlP5j@xX!WQ3k2~T zz|eZ%?hraO0NIU63t@J;dC^%b${?E5;#MuKoaYTnDsi>&Tu$bXPVWGg3e$(VY1z%m(H#XzJKLX5v@S&{Y zFjRWsCjy(TxP(OPug{bAgM~h{z|-ewcS~%wpCMZKj*P%oyu_@{RIx$mRtkIxZKfC& z1cRxQd@L8kzQrPMX5Ho*NY}QHWf;r1JGv6Z0l;t9Uk(*A>#C!cL>{bjSQYGi04g~y zCqKp*LV!kKb=pymC@S7rQ}!nT<;<~=bIM-$ZU0V&kGzK+Ap|{BR#ujT(@LG3U~B(l zNjDWMrmyI|h47;Yuy2qw>*_ua+ml@9aO)m2T$D`X>KCf%xw@KjR? zZ`^oAMBm9IpfBmKI9rcgpRSNFmhJ$}? z=oo2>S#NZg-a_90e9=6^Fw^F85}rSRz+`2TH)t7?jSZ7S5EVQRYwWFum-uSLJ3*4 z6oe~#7CdLgB*Qu0EY)dv8WuW7N9Q-GbfLP((Jl+xG%>zxJ1Lf>wXs;=+Gev(2% z(#x=6h=ulTm>BQhtE33hh`&6U z5GDM3l*tTWZL$Prt0PX5pazz2tn<_L{;*%lT)oCC1-<*#sNYzqNd6h14b&21iCZ zOi?V0OPJl*ECJ}@4#r1Ys_gm*olk4&yJPA}NrleGwt)5J=A`U>f4D`O*vYQ)T|>SK z+nFAv2CS@{b%;~~56UB)IOX=P{G1OsB(Ei(+soy2~A{i5;5Hff39-DyxZTpJs2 zCKs^RKQ}>##9Kp|vzjyBMzSomyt-`vkQb4aHM|$7#W{okcTsrR7a!t`k2VYhg@H+ylWJF=Fu+Mn6~w(tz&2(1d-!0tCo@@hwG?I1m6w$!H9P&yq&ujU^Y z`3k+)E!QED*(9Gll!!oTzL{7X;rx7n%8g@q6op552+Bf@1pZsSZ4 z%&e(_3A|yPSrrDTd4r4MC-uVpS?7kmm~~pG<+Rl)jp}c>E4PJsPCfauDHItM#et4Z zWf8xyP8DZU=Qb;I;>Qeodd!ts0LsLg_tx~cGrh-swWRprWOm7)lm^VP14c?k7ukI; zO1{9?7Q~4Lp(hJEX*wvgcXz?=$;ivgyLJPBh!1^{f$|fHKF~cxN7TxPg-oWT&z)M) zembX)zkI+S!1zfHO}k=h!c)b&tZSZ(^0a&;XBQtXq&+fVXd211Cue)9?W?Q$ zrv{{bT$}V(ig9H_p?z=Rk6)oQz&w{9Ky4c<8o zQhO`L1i6S;_{OHjMaX}n0`9vNw%DV*9oXI97R=g4HQ6TC?EhrzM|j` znD2~rJ{NfuDNx9u4Nk3i^4iaUnFDYu{qM#^Ff_}TQ}kRDd-{p*HR0I^|4IstU}>=V zc#5_(Op#yP*kDRKM2B8C>9Z;A{Th8q_}Rf>G}{J|8pQC4?V1$gJWN}1-oA+aSbBKF zYVv4>CCuY#cACCim{z+|PdM4fLf30VZ+l5UoqH9j^I}LLqti3E1rHrW$OtMu)V5R@ zEnpB>4?RDqL82)}Hnl;;GR3yEaV^?Rwa=f8|H_;!UD~oXfkBwanDAX)mbYnz?55PR zM>opc_f1gNla<>k9ByZT-cD|Ti`uV)E_wOh00!d+bzR-`?I8md9~NO9tW9Q}aICER zo)!kkPMosxM8uC)bgQ@VvD%3Lo z&D@P_*-dh3l$~xjO_X`8g4>6==o6|%M}9>1o$gh|^kv-@3^YfQQxN&n>`)jnv2h8=d1g<)D+S&eh&k_osAX{2lzPIC+Ruq)Q)~J$X1bdJQkRHj&&GCt`z zmC$Bp<4J_@N`iPY-tpyF=2;0{@6+Qn)CfvbNCFw}*Lp8mGOnPs*vW(1cS=a0TN;|> zaqB(9v)4SZ-45szm(JaBth>#HOq6ZJuob!k?l@tlHiKQ{m!4xaG_-5OyUfwS5OU`_ zJJp?*e)_SnWv^ZLuW`v8`eJm6IW4|AwT|s&)R~Mt-!q2wyw4B>0mGUohSj8>*hf`Vfd^&RdI3eFRC+3(aFnBn?|W)eU*}rVtr%ZOY3{Y z^9pNtU82gXMShsF6GA56%FU}IzR{t^L56Z!45xF|uPrgqF35T7TS4qgPqfu}Xugm@?u(~6mlO@PXHNyzKh8HT;hPkcSk z4wGspBI0c-6ZXNkQB#$%qqW+&6~1 zRIJbkaKCqRY#6@SFf{!L|DF?&EbrTefQ!Usu%#<}t90m=Kqw#v7FQ5Syrh9@&ze*m zBQ`vEi|OGvLRZU7iCuQNuadfsxF1h={h2CrMWb8JJv}uxEgp}3*|qidQt$ZjI~I&A z%24r~6=I%mJeuFVN#i8oz*~iM`!IPe)_Z%cx@d2le5;IBb-$I9m=_%BxR~XnXmKrK zJTY5@hO4+#CP!jY50!8`H2u7mjr|sn&!AW2cmDHF;Y&qH4X*~^RT-%se2O9VADk~? zeojm3)A@WTQ6O#i$9!qML4-hgE=?USQX8RxF{um!#kFk2OmG2x4%L=_fq)jnFjYtobTPFC06*-7OX(lD+AUhoF;A^ptr zvl&dlXX!uOBvvvNM)3gZ&XS>n+l{xf)U2UpWlV`fGv)zFVM%V#-3oO^W)=n+pc;~r z9hOqS|9v~jJYn;uFr$BErLFIKih-op_mcmZ77D;}^Ls{->XYHX_MdHfZmUrx8+qm@ z(}mz!H3u~sd}Z-fRV@&q)Y9Eluz75AgIcYjJOT9LJRk3uwZ63zL$G5~a8n-BeXI7@ zU!cRDCP2>!J=rvX)Zpxigd_WKPSIE%Ea9%0&Hyfd(HmnbB^>gF2#=6jQoiql-@vU4 zy?!!%5j>RJT{gCOAaxV_Bw=Ul`Ned`*3@wdMKTIFQaCNVK09+r1U5pc=rxDp5OH_u-HXUlMasXy@|69GyDDG}p~;j@CUqcL&Ni?%B;^hu)1o;iKW%_Lmu&k4)vBjq@GJs;N#raxVZFm)~=v|@SPotA3i{W zqfuEgz|FayPb;rU&l@-#()4O3&{p-&0y&4HKDiA847d?!dpbGkns1n9cu7#j3;j8b z3R!p@0u(?yfdR>TV0WmkJt!+CUT;>+E4+mLUnfmBagZGY940^ox<^#Ddsp?~vzwQI ziP}UC`YCXj`NQMd-<|u$Ti?tMqTc_AWI!49hK%3sVohAdXsKkkL2WD++i*&0C6Bh< z_!}U7ker;H2PPymHThTPrzIvo%Cs8 z=y3Q&xTCh3p|{p`)1bADJ);oPz1 z>UYcx!v>dEk!=yfBYUiIuXn~A9rcx-^H}zYMPHQSNb2H>lW*&x8?i&5I(&W(;I-dN z3?9+0?Vf$6%nR1xxXvoC$A!sT0$iV$oxJoTklG0D7Ww+Zqy?cO8x4epggnP*(MSq_ z`TM@AFvFaMW<4Fq z>Qt$pB3wo&4bK~>z?J(tn#vK)EQgOKFdo~$FlW$V;MWtNeQe2X#tk}L`61k+-Zhy0 zN0loYV0NH_Wd_O~m1yPqb?Oktdg9oLeu{EC+z#My3oLKn52BHMOyp-LC6#(tz#<$1 zt+s<5g)~?*6w4UA`7qqF;DiKaniy-UaPQ=vRvDSv;`La&pgS`>9dxu8s31umaa7?} z6uhspv^Q4j%N3fI56l}Zo%_k{Lk>i#K0KCvS#pjK>dIxen$xi0TwWo)Dubf9x7i*d z%svtVw8a&D&#CG9w$Vm+%}JYiD6(mSd0b)5y7qOGJsaueJNibW477QB$C+7hTj-oTONopzgke`4=vyu%^x3x9aBpj^m=Kgq|B`umVgZ;t?@f7qq zm@Py7WICBUU?!_b{xb!L+kY2l`#&3B&>&%vP9~G3q~m>Jzc}#{e?4cstd(7EwT${c z;NbzjsQub(9w5~zUCuC2E9}88d`s((pY9w4^Rt9Ij$)w!ZgNjdrn$+7sqJm%>zUbS z$fDd@?$<9P6CK#R!Yq+2VK&+Z-WW>5XkZ5U2GTTg&8@Sw5(vO+v3=o6ZcGC?lCwxR zamn|HpaJpS7pa^UZFs;BECRp?CxIy5(51(?m|rX6o7!r|rehglrruS4|9ph_Vw6q)hws;~$Z5Uk$VNPQ2hi+OQde)>0=QEETt^2N@LgMp;bWr`WLhREHUXkyi|IXm{;I&7wUh0hhy z?bXdL^GHU31lH-d?CoTU1x7QiPS)fA^IV&02tWT>s)}gQ1P9Iq^4QIb`f0kj1rLl5 z6mh>MmEI~)M?p!JRxy|zUUx0L!-mEWsENkKVE~ZLT!RQiu{^rk7Wiy4wd)i!<8Uc3 zJMa^F+D0{Mq7U61-wZ~Qew@Ru;cGWo5kDfG(wUG``{7gg1XN8iCDoPto?ctZY|Wfs zW%rY&7G8Slo&Et+pM7H|#SJOW5uM8U>Rh(5W#OX*3mF8C_94A(-)SeY{UHy|ju3$1 z7VA?4&x9#tl;z|i{QbL-z`WjPjF*a~d4ON;&>sRq_agvf8lwW-F*tCe5u2hYJPTK1 zTmpS*xuIcuVS>t_yqtmRLTGAg#1t=G;?T(8CN3VRyVp%YdhW2U-gYAJdt98cVI`T& zEQO*q6OJEA>pS^~lQ+5|Z7n`ZN>G2jl4#0RakeZ7MDYt)Vhbp$Jn{y1gU-;<8Z|Up z6QYBl zZ@&rbgt^^}Q|P2hS0rzL?pG6_*U7?XPFI|x%D_T2Gl5$Mq)opFZv%LTzKo+elgoL+ z5xMtG7F#}vfIBc7Um)WiL=XRkQKYS{?Tuul3fXI^MVI3&K`SSK@BVk4P^8y|4jy_9 z1AZn@^xps}WSw@qTPnu~jL6%}PacCnH!ruKV2Ui4A{sV6Ioa*cgBvk1aUjLAIzT)G z4}!7Dv%%7KW4J$E6o$vN z(Cl&&&!9#o%yMO9AI0>3oGriIr1`HGKV_ z46aBIKC?y^=&uN)zYf>O`zB<0RFds z@}oQ8-2cyEBVMlq@%iAUW>FBfPa%E2#`xe{qeGuRaHk#~_u^rAwu}FdueXkhvg^Xe z2|=YhL_m5dk(Ne9kRCv!K}l)p4n?FvVCZg;l9C3Ik{CLqr3M&Ey5T$UJihPy{l2w+ z{O5Abx$pbL-q*hNwfAY-`>--vk4o2tpuCWl|Nj}vujRKl(0DMfM_D*Se?>Rm#w*on zHXn@2>8mh(-*VB)u2DQuZoMLO)Kg|#0m0)&ft@<(KVe0Y!~Wmp$NW3Ys>|B*GkoeQ z%=a9U*gl$$Q`_EdVsppnxF|W^DH>4s-@l(s*5DXhY4N3do+LU+P$W>_SD<|T*p)6tJm(P||OI z78V-BlzKp5vgs`Jy7I|wU)AP5P6(=``Fk|UF7%tlet2yvDR8y|z)r>L<1~*K^T4y_ zyH4jRr5rr!tNHs3EuQqHSAy^4*dzT(V=CWZ+__cG0Uqkt^7@?w*hk3gOrU0Mq5uin zdxaOMEfJxQ_W$RT=z<0R)t3DGEZLcH+nJm&KgkEfQGQuQcIrtP0W|-yN+7}l*=*1M z8Ncpfk(CqBbX@-;dYDpB`B{WGSTBBbNZdm`xuke=c3ulMhGDBede>XA3}l5ZhTrM@w?wC#{hhMAem#V(n|)*&QGZLG!Q(x-1K*fvU zuNpUd>h)Ivz({9oKc^%Bzf`x@ew3}leWW{e?36D|`A3uq=l+I8qpkd}4gI8)dQVpI!q z65KhyvF;K!&Bm^pH6q9b1s2W$|Fc#|x!N&r_#7Aal#Q&Am4_&9?*75c?PZ-J@$}2P z`O!Cq|B)sYkWH_vTaFFo0wG5ba(+?cM(c2}o_HD?_>*PU#na1k`}FjL_pr&~=)$XW zYzg zt8O|Sod6}LJ+U<|v-t?kPoqfl36-b?JD@}BummuYFw%R!roDHecb+Rg2RNLb$7Ks_sUZBRp zu2gq!>R$iSBJ>iem>i0eEBH8p(w!q#IK8OO^+`rQsD6uljsKq`pS@Z+jdk;i|LCeJ zH9@t;BZx6rq{^18Z`#e?UJ6#CK-s`L!GU5jHa#&3UQKSfkvYU=`-hDDVeRjI5Nggu zv*#%xOifth6qLexlh^Xg4`a<#Zp9^!g>9xvjJu}r*r^fUfCc8FlBl5eN!@}=rpP;% z7Emfzj1yHF<+Ht)XmXC*P{A+LTNN6TtZ&MWpFaly>=ScVHW`P_-S!!VPit5Zm9pes zoMr0gDl1v0Ar_YEdd#5K;|_?p|Fw8b&zt4BGT}bpyuYHBH?6r=R2hbuK@+jlbsJo{ zF=4NV%a&2(l-z~I6YCwhmHM?Rzj~OO_%?ktf`GFHSXG{@P&jc7O`-+6G>Nj~35T=#vsB%xV5*j4x)9LCMupTR;r(1cP z33#-KVob(YTVSlisbbW7r05%JL;wj9H4s& zq}JMes<26vz0lHqRlw14bG+2!{hUPe9HOCTcX{7|uhm;RY}_fCQhjhnrK;+2Bl;61 zC`72$22+rocO}5TJ4Qgl>#6R|GG9YCOuw@X)c-1EKj6LruBh=B2TD$z7^HS$Vu=;F zy_#AfkFEQjmHUD0(S7i)jvx?1Z#h_Ar_c~~{h3fXVBu$$wmt+ry^LKMJbnu@ef~*; zu_=$G%YPJ2^d+)9BD0zvNN{U=WamKa25Ie!6@sn3jUZM>>t1MlI0KIt`)zqZSa%I5V(k%P;FvIyYjHzjgIuDWnMGydIWT@k;r0X&&b0uM<9Q z$7}s6p#%q_f}f}3Gw!xo67WvR`%MOB<(G6CU;CwZHcBi!WP*ZuUPrj%j5a` zF=*}x`honiWx4S}nNmW)_TUQu{`F@%4=H{i{vp^tN+r`J zJiRE?Tp%cjn-oLR{K5aY?_huz(agy}5f)u9O1H1a5&RCcM5p>8!s-4&33~KasjsS^ zJU;dJyphxCwOoyrezwXdZqFYXoe01_YL?A-7H_y7l`nt5U4JLcTTE2gy=Um+p^M+e zVCnK&4x|+9#cICbMdeI+&Iuj#ccHRm70M$M*#gU6*C$fKHtk<(43f|&89rge+3pUf z8JxxvMz!No`J| zc01}=5)m)r(Y>!Stiax_e@G1E_<(o^ySU|I0BCYS2JTKasN`<>^c6G`IU%x*e`0R< zjZcylU+Y<~rj;kb7<}A$3;Na%aJTQKUU>#aN4_8KT1z;w^kGysmsfy7c-?yQ!3G2I z-y6tS;WaMEb8v8oSCXfch?QlXZH(?M!h<@y)IfhKDV6NhDM^2!YfUL6xeFnEiZeRh zFgU|$Zmy6C_G+SHNwI0Nn5|8FZAHE;R;^DXFtg37|>{RObB!g0h3xXdVuBt-wcs8x9OBP+(qnqN3}q?&8gb>Q*|)9K`f2 zNeM|8)og*Hz?up@5lL2q?iv2MM{d24hLZcWfYuh3e`9eiAVNYc05Ry#&)=wDT9bpJ zq34ZclrzWi3B+!hx1#{GZNrCi?d>Uy_3?}j_G&4*rRwq*k`w1k+-~`OWcZJ^+A3(tpqpy<(+$}^`4Z`sXXRr2}(*|+I0b8_1@Uq zmni6TTmKpkC)H4!_C1*+w?cvOfh6|d(+7%cICcb{9BD`v+;7*($-b}Tq7bjbz{gQq z((pS2WYgX%RIrOpd+4VBhfGHRGF@I<$HUrZFx(nk#%``Q$rJ^29NZ&nKMMeBJ6%@* z5^_U-vcj9`~6ZXc6b-Mc>0@Tif!h_5x)~Sx-?@Jci^OD2H#$I%0WoGV3hqH5X zwyz1=&U#+eQ4||AdxHie#@)$64nGB~CM)mKs=72*nDrxwo|+&F-9V)%7Ar`JcUT*h zX`X(0VQ{l;;e^m1IC8{%DBXzse4DTQ7E%|DD5MG5LqCdjV5lZX-htE(G+ob^e;K3L z^?qwpR7|cM>G%^+465}E$~8lIUjNMhYMmo&B0R}^GTeOits*nBHO9<<4ZV9(WVYtb zQb%0=dR@ss1}T<5wqC?n}G4IC+=D zbZ zVFJFni=w{O@ zQCWJ~>dTVoaNrFlUM9SNiQ0t5R*+MUy3f6O?SKUiEh~CAvQXm>YcY4}J0(Y_8DG^4 zn5wipDlEU{ab!}oB3abcq6Kq`R^`JtWs^k2F?!BRk5CD+pjZ{!fItoE=|x!%R52Q0 z7(WToAMrW8j&Xm%;8u5kU`WcgdV$6#mzP)*R#sLux+V5XC@^VsSbm{SJwiABUb>V9 zH5nN*Q4kcAKFR{p0(hWX>+L9zDG>y3u2O3L@<_vyol*P+L5}W2JdS8l=gPW!%M*bF zqd1Ql(0@MaDV!oA*0Qt1pc*Uq_~{h+42J2WFbT3yvE+IkDXK|5_M>L+OIz;jMxEsfY!tagly1QKL{(jX zn=QI7U><--fO8Bu5_h1Ak_N1-J+GY8!|89N%f?D{dD3LtvpE^>>@!UOfk_?5(z;{) zvJTLsn^XoH<+sz~*WQ-$MDV1;c3l&c*aQYZ79SJ%Fy|rNs;0wqt2<4l8G_|2{V{j_ z?*dlwj*&zd@gR5!uVo;S!Ca2XTsnr;B-7T(mLE%l`V#ctIe>? z&ksETJ~w9|q$7eN=Ow6|!iO%0%LAPK!hf9w3M?C(0jAqW>mD$dY!wkPVFpk|YyFW5VPg8&1nk^2v$x$q=kqbxdlXRZ6%GlVX10HYj_&yT*Agqx3f=v!WHS%RvnrA6tziA2ho_Vo4|4}pv1WM%L6A=cIk zGXD5uliN&vgSWT0PfTceysmUVK&FVeOioSlUKNO;>Qpr+w)YbK;F z$Cimh=Nk_HXkVYwnHu1FaNjuGf)Oxw-m)(DjmphdRq>)06b#UM+bb)tPU)p%5cPH9 zJj&2n4hu}e4@1fldM5zJ8yZCQ-@M-`d6d=vJ8ARLUG zthkKR?Y4)&xOMBLhDNlq5GN-mxbqp5f@_Qu?PF7#d>7G^V!uKUg0Zy5BB)+uGV1^il&CPPYrGyhqa9&rxNK zmf4nXP|5N3_QrDgTpPrt-=GPKkNVse+U`2Rd;0oJclYbss182m<(b*oJaiLI>z85m zINDP0i`e4m1C1+8X5Rl$X`aW-s%Ebf>5QP=KKP~$ECW>s;tpi+cH$FMCXEMK-UhvW z)?5Y^Telf1{QO3}eu2u&&veG7zC0_^TYUiqA%(L#-Arvp^zVXYey#pLBn61U(NjBKo|Dy!A8v1FzxLoU18c22*^^8~0OGcqz5 zeTnbfT6!3)oObwYQ1szLo5hZ>zP`Sal9Gdi11x3vc$CSBiO4V+_3fE@7vPtH3WX?;!2qX$D{V~3z85u>Hi>D=tBI3wdyS2T+;=c?R~>X;)fSb(Qt`8m3SP6BU!r%xGz$H98WD&%uYru@C`_cpH5Ef) zWs&;WMEErP@j62G0=wr}fLu~mdPm%D0Tf!yJfQ;pl_;b9$GzfWV>3yde1Uvsamh&W z#fv*ygW=R7f23nK+xCSSqq=i)aw5$*4VrWri6Yw#(N;G%!-$3MX)^~bd#o_aPCtv%P4;V{2^b8EGpiGZeuktP7-NeD5y_NpXfS(;% z&Xbdq3&dSC()JDx6=M(-7?}y4!DKgCs>t@v89-(dS9{Uqr3r6G^DCo)Yk{ zAYq<(gBG8b7D3EFPfyYp93I{w11`W&kIltR-*C8@Bv7K+6__8l>g4P!U@?z?C-L*O zz6G%~l+*aveHRxO&wP-8Jh-bw9@36gMAkJr8e_H89zverh#^_<)d)ouoHudc-PGaZ z7`{W9y`(?cEbn!OQC-ak&|WUFNN6>hI>{$}Cm=;lXG}>>JgIKJzzy@i77;&bc{4JH zX=)s}fD3ALyEssng^uJ`dk|CGX3#69n+b^CJ(xNgAa5I4n^U>^l4%HJcaWU)bdQoFuO7_F3sHk59lA!JQf*U>2~Ur`6S(B=39jx~h$ z4oEe&EBl-dr=2gcfE#31+jR$(1g3PWw}?l?#zoRIXpxeV9s~dMXf3K7InN+{o-I%7wLduNL}h{0-q$BYk?YWX z&vDwm-Y$Bmub}xh>^V0MYHgK;0(-0-&Q2$S>>i)*^BL}<#+SY3xL4fcK zLpr_mlh2ipPwpkt^Bx2?q)W&T5R|xEYTs#Y2sl-UY?%sMM=7Dh;7#I?yFeGc8H?{pM`1TvhQJ~u4TK5MN9Bd@6PWmZTKiS}( zo{a;yzbjigyAIM}>*>*?>bUG0C8lH}ywrUPnF?}U*_Yn`{nP{H?@dCSylDwiC-e_f zPL;uJEr472aCNkO70^VklzOWQbp(I zTOMY7lC|C!?z>xgIXSg8HNQjX$@3N;Y>#=G2nc#&4t zyKA~lE-%}}3*?Du2~N9|!KFKC7gqE5AxNsJ{rc5k_fc~^@w)}xb^e5(5<5M8tJ#*9 z=G{FYJH66;I$9b|BXM`}DM%Pyue5QAFVRGcIoWAb!XvkP2Wf5BYn;1n`r%KjWRn63m18tuAe2*NaYwQ@buqo}5RIIEx z5>Q*q!73-aZqJIrtV9#zOOOe>Y>>C@@-j{o6ycfN{Uo(#E zp%)cp8Z>-Lfp`G#d)*7h6bCZLY{qlNOh!TW{33w+8ZWM$UR|6>3<&<{;@ui)PK8W@ z-g&YG6yRba78VvnzOba*;fB^OOf63-k`$gMk(SI7h(7)p03fx)KjMC&V_>}=r4SQK z61lv3^@=UMGAb%+XlUryFHKmOZgbH;6Epg{9DrsMGqVa!4p^YOYDS%2z4N@9t0)a7 z46CJ$l#G9!5E}K~a=@}Y7HC@|?d*Rt%`Wi`<>oT`hX55f`9s|ohcuKuce*+{K25(t znSsK7+y`;n*r>Jc4yWzh^%WCl5@!=W=bb({6t54iJ>mA%apz|`gR3o<<||MNBJ9}~ zSlO#9Hg?B1iv2USh&K|nZNAL&In8SZ(reK3>so+)A(7^DIqHiF*~y9!PvrMdvi4$s zBnQ8yfPt&fJxVtgwLJ9Pzjb)z<0<>^{IQ-6lvS**|CW$Aft`{Q>m~80i+Ub^{lP1z zqCzPCoI2gn(UH;UVMzOePPI+3c9q4}bZx#Q&sU!APeeOGOC$2x0|NtK`GrM9vMNAR zEfVbycOS4lfsichA0NBw=9ZRnxpkj$>}*~8&DT7Ff`U9(VN`oK`ObRAfv({t+f2Dh5Ae3YF!$V7 zzYU25rR6_)vNAKFiz;(Ab4@AJ8ARsbKaa0@em&qh`QQ8iEzvryCL8!6k2hpTLbjW$ zo3@K1?^H^*SPU*`7E=K7^@KdfG*+n6jyK}tki z<2Z$OCjR8Pi>;&8`rqLO4-UAP>jW$%3mR~j!F6eWfc~0k2y|>P=Fuuu7EMh}UD8?& zC`06rEwzb7p+xN$H=+l{ItD5z$jZ{KBLT1Fu=moCM@L7;{;2mbd@aolpkS3NfQjR* zFnQNhp8!Ei`51e;l*?y@l^s_Q&w+*=4;UVB?;ag>=49N9nfm0H3X~)H7gho~C%PD{ z9|usuM(n!ZwD0}y*Tv`CIm=bOo{dG*^dcSs^{7u2Fa?yOVsus)*J;z@r#nd3O_b97 z(YwKsPOA2zkq9a&h?99Cqn8-1VQ9kt2f<03>!CAsD`;D_)NQ7h9g=g2$8GYjA)5t; ztaa~R#hRv^v(N!;*NwAkYOZ#YG`KbAZ7YAl9#jXv2h~2x{`{gdoLbH81uB$Yuh#xK z2U(%V*!+&`Qm+WdOYyXR4L3O`lt4%&lF>IjJe;k%r;Uw^%b?~Rot6rS1OP377i~RV z6S?7R0<3_&DHDVJ4*%HU=2YEi3F>wD;qRsIVJ(22nKResf`36h)57t zJd^&R0=jB@9RH!4D7Uab+Le+4bsJ&oh7tspP8YiR+T|17NN)iq0O$UWd@XjhDO}u z(8^!?6cchbCW;&1+tM-*x_2!vFLy`ynVXv@3EIB0`ThX|n}>(zXR=|L>s*u9*eNCq z)Y$@RfbsM5Y90`g0qG;yUO&_!Nkpe$eEf*E(bg3l3I|&ker!IVov< zVIeb_7KQ293t(Fw2dvGF4N8!iN@7UYt#`KWU4M*zAcPLFojGJeWvaQ>eE?TC0#?sX z3vTOkBoera3!r?gDc80nv$nQon;OzUI`_2$D(urRK&N^lOA8AN8yiJ%RsjLfeEl2* zJOf&w({IZYmCC_y)>ELjx@Foy;gNZ&FdPs_Gj&ejWv3cFjuQDTUl0_jQVCdnAI}dp zIDm+Vh>WCOoE>EalW}OK!&w0ju*w$yZF8nxgU(8&)c)PO$F6@K2$QRSK{a020|X&_ z#&>ttt92C3S2d$wR#Nst?6vei2DU%~MQ-W)?|@$0kr+z>)vJ~p^vQ7C6;j>Hay3Tn zk?(_>5O}Q*Go=H(fhh`%4Pbx1>$D`=psEBa?5BKev9WhSs$H|(g&FU)bHhcBBnSIk zjQc!GJ2AkgdUApN)A2fya^3N(k`K7n&_*@*Bk&cFqGraQBmKVyECmCOn9pUJm0wr+0B&!@=jru+)!bIt9#Sus_cdDQ6Sd}!t3L7K>(!}5$ZRg$O%g! zK1dtB88W1F2J3h=DmOTw zpez~GV07gDNd@ozxzj5Z>Vvt_>+!s4Z709w?WuVUd=5r3BjL8-xm>sDIJS7zIy zzq26s#64B%A3S)CzD4vRD~lv1cPOcmT_6!dMG9&VQ}W;gNsb08DZ#BZy|o9=3Gc3n zWk(T2>B{>u&Ixw?DBGCT!(vR@6>VODCr!iqdbegDHlK-9sLe8_SN4G7ZqPB)duert zUKnu;<{u3ThAK0AF=0QLdS1yHStG{^ef}J4!JG9M=nd&Y6~69!El{$ic+Tf|%d(Qe zue?~xh@faNtUHjNq+2j&x^#{N@NeH&uw~z~mHaF5QZwus{|mKGIq{;S-uz`zri z%cvL|+zp;$kZ$Rk zpxNF`=tSZbgQ_+M9cqs4i-zlW>slVwI3r!d0L4IP@Vk(UXQ^(83H8EpGClTNuwWx4 zNv+5KJVPEl(G3jCxZljgxtXS zT!^aK-MbL~i_^wjC#+l0rzhs4r}T}dcOdfV|19FXsi}F2I*O!FmMA7Hiv3l|=Kfo= zl+eBzckheMUIw2t%gaVQ!m?f}l7xo06Q!jMV^_0h$jPl%OpR(g1I2K8h0*i5K-DiK zP!M)#?d^$}OyX$xiBJs%RedNa>wZ9AT}tDT9rVLoYhM};xa zAVHo&rk*MWUPE5|qmHtBB1Dsw7%+|~h5+C4kMfdtQQ8&U@FCVT;?5RB)x#b~Uggs8 zL}3&Zh&zqN_?x9X1DnZjEUfLBC(URk&zir#vwNN9Q}8zK_s_iOyxsv}Y&3{n@?iwq z(E7v2$V^ASm~u7_I-UUM`681 zW;mZR3llz#NR|`*EvPC4*-n#kr+uw((LiDqB5bp_3v}s$X0}J ze+mo?)O;Hj%MNIiku}G}l_>ioTCs~=_R_c11gLL2^0|rW7vb#=wTg+Sel_U(9*xb2 zYKE6OBgA2pPqfZE8x-Wg9?;GhKF0$i1y;*xRAUOK^szu*?dhu8NV+bVr9})0GowLl z^j+uu4aTfE`YboIA`JpXz8{x1kDlv)Z0AB{nkBli33nibgoHie5l&mV!)w6!!(y9ja|}soVXvpUsy$2IZay;8hG~1z}>d-Ybo^O({(#EAKzA!WNoq=!rQN zc1xidPaSp35l8eK-ghB8u`jN_tA+QUR}yhvDVotg%Ehn=r)&+}HGnS?NpUEh;kzsAo;?$% zC^-qcLnvPg4t6@}Ef|{B)O+^6ufAlK-JXKNZMO5+*BMv0wFftprZSL>lQf3|CnnqKai zz_*_UYPvI}-$E&VkS^6`pTWd%CKbO8h^o(^DUgWSrk?m-1g+Vn+o-t*nZYA1{%y_$ zc>KGTRRAT2*Pdpn@#8BJl)2s)^S19p)AE}eJ}3Jan4|r7wmHZXhj=?}PiD^w!)c7B zWZlCp=j>HAN_VK+xT5qIMchMeX`SXjFfr)t;W5mHHlB$W46nr&X%@P@{Ser0RF4-C zSm?dpC&Ax*^}w(o_CFu#pS#~%`mJx5Vz^{JKfua8VsrK3ncAv&iQbE+Max8}OtChC zRk;XOTxiDO!Q_;_6|D9XShm};&x(M$YI`E_C;AmLWTI7Lu zC_UHPTLr(HyiIGtRdTd##x-z`@C(e*`|qbs;zCA=poR%{C;dY5%QgIO^W7Xj1U^Uh zTMC4PG}<(tb`^*|qa?I__1~X>*=U{>m~yc4I8wpyIiMI-DmnX=?8^gp_jhV6!}(Ps z%bTJkXpY2BybTG|bOdfGh~4W)8*Ad613?+i&8B-I85Fqu)IoYoaB78*^6JH+T?(U) z=V;DyTghggINec;?HPx74PrccYW%$Mnf>TaagcNc;zEYwWAlo5(6#-2!)=~u28Jh5 zN)Xr3Ma+gJW>fI;HLpA{r2PD!84e-mq+A(Rh`MXu!M9FoEJ2T zze6qBQ5CnJJi`6pGFga0e(#cj?1zn*J2+RsmKL~$eh93d{u*DYU1b?!>Dh`%NnCf{ zX8EfHn6oA*v-6U+~boL0KqAdty<-ridum;SCK` zGHnTb%IX9=h?rMfWW5HJx_e1D^szwT2(Zw#O)7QgPrpCh;;?_<)F{SMDM3nVtkBPw zXErg?P#A~-dPQUdf<`!{#G}6{@iu#)Yc!4cnnsm{y4`b^^+BDzDQx;!?*9&E(f2c0 zLC)$xG2%9}Q?rkuU{`E(-PVo>f|i_=f}E4pTUj8_a#p7et=0nm35%3%)v+!@f;z{o z;X|Om0C1ACzu8KbbK%;Ix{oZpZJ)I9{6EO&dIP!Y$xAzzvx-f4j$$)gmUm5>T5$E3 zZO(L7M(>~3Ext-a56x;*U4fZbWD8{g?=dK9{G{#2$-6V91-25MQtub8MFo_?x zkbrNq1ZsW*=Ev9Y6BvOTUqr#D+i3q;u0WVkx%OF*4ObHA|ALBBODsO<{_4LFC9HMonBpzYyn4a(NajQyC z6q}_5K8{A5VABil9u<3R7pcqZ7CRxQomN*R66eIgcJ5)D{nuK%{dzO0rf)Rm5$gT8 zQw856)bff77#DAHj+3tT_UC!SZlLc;|HK`*8imx$v&~AOAs=~y@dAU_9}!2mQXG5# z@$85<0p`m5lye1xHQ`Sq1~jVC|1@2EBl7L29zd)ik8 z8i0~~XgEFf^|{F8!#5$==}a5vx0rsZ8v~fX)JJw$J>_(UWwQTu<(fa#ix%L#kg`zo za#N*KXKf(G%89rCE%NN*N1r9lz=K$2jnX-9pF)A5_Fn@ti=~TK7TzI}ZT5gzKzWOS z%0zESBhIr&8KDBUN5{bqwi}|B+_rvGAVmqBjL{5QuX`j3MQ)mYn%>iOXMgbIL#l3018)QS`5C<)3MZr(Awy?yr`5N$FY~a_$M`PRa{OV-Qq0x7u4;^N8gTJzkHH87mbnQ$OA_NIExU;FrJ)d~W?T%C7G` zN3Z^9s4HKuz7M^4O8jD4zE#>&+-oe;De3C*i9tR>3*Rt^pHKLp|5;w_MY;ER+x96w z%}|a97rwIX3BS|0p34#K{??WRzPWc71HJaO%@P3yund+^>0ywrzis*GxUMc0&Y!7B z^_((;QzvO!1n84J%q zymxzYs)<}b%=n2qhlX?~lX8+!-ZruFc)pg}3JL&@o5G?)JCS{XkQ2%EZ!TkBmRF^Kfa|c@2V6o zPA_e2>>S7g$5Tc;f%%?l@q%pxMH5e^1mh6dnl~*5CR*0+`}oitTTW|48+aIfXNYjN zbF%ym@kB)t8nqmGJJw*5)|qy1x8=`CNbGHYDNAo52KveWz%coQle-XjuDmx;Ieg=} z!Ek|rZvWY`IlZ1?>3n*-d;1(xl9Z|Ol3*Kyzh$+*P+>$t-aS@tS>CqLKzXuCu=n+y z$~ELYP7K&?u|gbLLqnU6Dy2lD_}3?XA5TJx#uN%IB5jfo`$c;N9n6cgcC-iY!m}Lq z&%azWtF-*{^cy32gYu*}dw&YBBxMG8LqcnFkG-}#8C<;fchb_lntY~j%{SQ*pG(lV zHTDK78#dji7PHt8nvdTR$odpf^XTI2*tWUHuDcREwU0~l{>gIuY8kOTRwxZQ{{r!R zEthnPK@H=Z!J&_7ui8sBSwr<}5&PZzSCi+<<7!j_hdSc>IXT;n<&HlsCA=SORG+u{ z);JUe`(1y$iT-~+{zs^l(Y~93prF5eqCm>ZGJofkj(u$P(d;MOeti!O?Ff;LD#7e~ zC9h&qO>iQD_$WL}*4B~86{JNd$X~{@eQLTi_06Yqk<1y;D3#b+<{xi0D#=?>Z)}L2 z_G#wk)H_?ed8FpXUt60=W@?;=`TTh8V7F81^d641_TO0-zd7ju zTT5{^1o`ruryP-}PvAn}1+SiGfh!F48!t*q{@or?f00bS{|O%Ch!V%`HthYp3T7(+ z7Sy+3^Iv@J(+bzEw16z`dcHJM{|?<|sM@6ejbDtN6GH#}^`Gbu?YbAbdU1Rfd{eEh z2oC%Rznf1;hI_p;ZmWN0lP@Hg9@<;nDI+I}`{{v;i^CW7vU0_r75za7&r;UlO&5nG z2P4VaK(p(ZmEO?pKf5}lwZ<3P%qepIZL0x^*u1p#!Kd1HetY$vuD_Y!^hd~#&*TQ$ za}6h4TU*V1R*-X9a7$ddyoL^HSN@fk-OfPS(uE8o*PZIk;O#S4&3^s)+F*ms)BP}E zZ?@CW=IOxkxlrT~{}%`HH%mq-M#9dg^-k;lG&I;VGfd>JuIIlRgn;X~j-!?T0;aN` zRoP}2{FYl!xk}wqQ#B5DPjlqSow^1BXvIA?&7+zQ)&pwln2WS675k&&WObWaf4#ml zuwH8Mh_TYv=a~cLgn7YOdunP(VEn&MFjGs!fgz%2C5{8%!3j5CLwTs# z`{Feh*TFPkpg;(@`{1u1Bdb-Fto`1@ZuUG;cLgwjdZSE!iGtM_F!6)++qHG68n~ce9U40U{5|DDVjBY=HWs7&!nHg z{@SvT>2=iQsywki@NE;bcjVk8S#g|cp!zyQGK5g9Vbx-X9LHVl5GAzk^&Z6P!KGqi ze8K`?3V}IMU2I`6rxL(Yq!P5CeuW}=avdRtOrL{&1y3ZEm1Ona*~R+RG(}(7tt4sw z=vJjUgZ}t=Al>yGTf8-m9n-=O$ZqKRMM znuEKCz8HLkx8<-KPF96@u4d%xGJx#()45JHV+_^vXAh5W6?v~_#B5Gp?Em)bnX1$p z{8s$v?cysVg-mV%S=D`^`FhH-Vx0g%Yc|TU%;&}{5C1*0eSeW^*}i$#Po-I(rQ{ZZ zqVd~^dzs@h6gVoFerbi4(6|zAVrFJqahs;M0!a^-Lm|Cz0$USWo^sALEi%hop$S-} zoh7t-D+BY{KvmzJiv#!O>9nCV)RK}FnXtwT{57qDJg-(=JwFE#E|R1G(1W8l4F`qNm*ZTK#x4F3|4iDK&_Akr?80^dAd>mLIxMi++^ibmNytHm zFV9ya76UDHZ`^xQ|C}9q{b==ue}Xgl49bZewY;>L;w?iN9UQYqx9aMG)826Eq^=JO z+8+mgRJR=dz(C60ubKTMvX)JsTms&eyOhOZWY&k}vpkxAxOp)%jb{H$)?ng;K%b+N z;_)e!6S($OXPucvdvtyHY5&Ash=~VV_s8bF)P=XYxDb#D|LE?{mNu9`1=Z(LYi>$Q z0^tApKxHca>)+Z`3xp;}X7dLpJQ04X`U&-Tp=tWG8%X3f^4{rhzd^2M`>p8y1yi`< z@$&fIx&X8FOhS!Gz@PKUIpWxh`@GhsBP?m{4HFW;wEA$BO8U{gz(QSCc`fb)-tVyf z{1&X^7vpnD%Dv0m{&yjb4J0z5;Mjw|!P@^j9x(KV|5Vz+#l!Px5R@|17m87F2$*M>cS7{22T_*l`_wmvtUk6K) z3AS~JVjnJrqd+x!)Hu4_l;Jekgx)RlwWv8SJCr07bb7Z)Y}x;r2}9uVK&&JT-}VjE ze9Sr4yTriqA&HOGSXxU8Vd^ulm?SM=OEa3;UVEE{K)TAt>&;MgZocbc?Z-mtHs;M= z#B*O_Tm{RYhXk6!?X4#HI3J0c1#Q+~YlO*$(w-~5UF@LS+M=VO`2od`aM*6Nl~ZWk z!Z7M)O{2h%_&tp#WCs%lq&4kESCG0yzO^Y=!RohuPj+?vFjvm@)k{Z-hclml zOw!42c|bXr{$!!Q(AHJgRN8rA8Y81t-di4%5{Fx_o)yW>q#N2teIE$~w&AsfA6aO{ zQ|&vh7Fg2?Q$?vSPU3`!XC1JQ>*~)0ZSD`05mV+cxNJ_8))R7ZZ9g4992xFsv%tO^ z5=;JgcYH1w&5yL=tn8!PN{UN&zXs?S&cChd?RlE8LXuWF@F%{!%o2DIjK8kG43-pW z9U2_5I?*!@Fa93!i6haMN3&E}+`W~7frD~l&_EFjmX}{rGT(TXkKX+q0bjOszD5h-WX?ro23KnY*|r!YRsRW(n*P^-B-6^LF)o!ZzNg6(qXq@U%m`5|G&QhHn1l= zcE5o_m-u86{2Q`Nad)*>PswGvPZBKN7VDJ-z5}TO40}c^E{HAu6c1 zU?+gmOwo~36TN1pr9cYs9}2;wXZq*%bbeXJ)|T1wT|MsGu-oClI+t=~ro_@59SCON zQk~Y<$zj&j%QWwow>x$jHhHRpyEWXQ(ab_S%lv-0pV~M@+UiX$a|fbQnVcW#yJMPw z!n2PpzgLh*4@IBbs5#9=p|*aQ8D9>Z2N1=$kUfAmLOOx%xW)~C+48pXx9+dL_`FC@ zO2TAcSqk^WIo!tqVvF6~x-Zo(=e=4Tt9u=PD7q>?=kGOJ-?16%Ej3q5LfsU}K3|DL zQo)eq90_C;-%jAwu@Z;B%4KLtgAKgbCM=Z;UkPr&^l4|a@)quBhEir;ex&u0(=6Ae zXm*`AVd@b)@UWg?FP+q#+pdbfI$Q9xd0+TYFY>El&S73@HYn;eQT_i+n3}xwH=$8? zr{()6yA$mcYTe|M;b~(1^l1gwwNaH$d8af^RTLaAzpQCd8F+Y= zR3BL>D|ZR$>xkaP7_Uo3RE;UNhRqjtU#zD%BJOBy@P~ zux(Qg{knxER@x9GA%UAaQleSCxg<&08c|6p5opkMyEQq2y^aD+E^E2FZ{`far7LW{ zJ3=Z^dU-Zifl*Ayb9$c3DbjAXon+gr^UZKM?F=uEf;ady^Z~u0&|iRArjQhw;(6@q zI&SkG(=WnVIftYJp3360kj*5RjuXR@hxtr$u_N|nVZ)WGq)_LSM|eQMk`;c$ChooC zX-yvhJ!3XxX)arVgn6n^yAmRU)#M{wgs|K z$MxA(xANE(zkB3ddKquIxhEL)B|P_yOJ}`EY=b8CeHJh`81L(JC}nH5H;aF7*h z_0F4@=$IG*sSf2QoNkyor^v^f#1Hf(d|Hm% z{#+Mjmg+V8ZM3~||Dj&JkcE;GAm3P&6P%+rn?`-o7urjim7E<>TC)~N{i*mZbko&d z3#N-YE`4kn9UZiJRjTi`GejEB#-S@IWZOa{GhZs1Y-87!yy1WRBCtOr3(h2Uflbck z@bcLp28>F;wp6DY6hDfM)rOD^PVx(HJp~JxvYq(ws1DNhRt*@f`oAG`EIx0!8v4_I zr|tP|1j6#sTN z|Frj&VNtJNyC7nKiXtIOY*AonM7mJ{0fC_#6s4rQLlMCNK>B6$J9ri=Tbf>OAxN*3^;51k?n{Nw_U<3QjVn zLrA9&iuT{sq)FAS?M17l{3^f@!YXKv^3;kJ)LKlg*+AG~QLQu8@+NoP14T zCmc6}sZ!LIX?rdqtV#zLvK_zEX?PS;{pAk05$j^D0zItd*mu>n?FlCp2~m!0jhw#T zA6Ww!uCz;0yYei=lx!MEZ(QK)2FK=s&m6QxC~;pQ0>px6-+$9W9OjwkvD|Strs&Ac zcoI0v`Rg}5kF^S(dv1}e^s4h_dxf`#%Q15pUTScU?W!aWX*}I0W-7tv`udY<`QqSO2NViSdRBlixDo;D1lanYem`>TFZy(=$@8b;mzw&;^_vTzR-x(bzR zTEo%D?+0ybuxsyJ%{_wTnzgf$(nk4;!?wiYu&4UI#@n60*dictOOtjkmBBlM>)SVs zVV$+|^%DxO*feuP=|#W){r$&uLgCa0G#qj)FLy(qifX)&1&Vibb8paU=yXt$qH-C% z@K#RCdTfVN-=KV%S5`+ztd5`X|snJhOPx`|Qy_$`(J02tpE2@qsRFP7eHnpsd z=k)jNoI+o@-K4T6tSZ)6`gAb&=-KyXkGoR$n*Jo?fi5?xx^3=a-SIBuIN_|jlGz+Z zwo|GiBqWmiQ!#B-inn9?OQH*OsvZv;z;6%^1($+9rI|{QHM*eMWubQ5k0Dm zb!}ucp=?XX=uVcH^=NAd`F-zuo^(QvaOSy;B%8uf^_CkoHMLB!uoE>KijpfOU%zPb zdpgTZbJH!Fqcs+6#Bst_*}FBebK11oV`WybR_uDTTfJvwn8w|XauDjygrMc#ZfRm+ zcU|sn5n)HMnt6omM`RnTnk{aLqo{@t>vdJl2Id_C>**ol`A?RvBadd3%F1lI-0+dO z=c%jbnrnT-!arXWeY$jB1r2m)v*9lnmv==1^h;RJN5_|}i~?u_j>Lqf`qhuT+X9V@ z2sg2Y#!LD|WG-%k#_`o5fh9LjMup85ORW9uM@7K+HIm9$+p{SR_k8-x&l>x!{?E~#GJB%mSJGXi%P^W(;dkq6lng!6(~}%I>Bvup z00pVnEl!NBjn|P_Ho|)w<0e|9WIx~fDqf8Fc0+P{w$mu;o#5ySDp{Moo!!g>Kl`W1 z>Ty`B!1-F}zGP7KG2QaZv(Il)!p;0o2Y9C3ZKxPitOv4-+u|%MWL&;a*6oi?E)UfO zO&b~?BWpGh^e0jTuTT)95GfS#Bg;KtB`BF{To<{xQ@V`?AL}s1B7l znlAah=dWLDrtDh2N6qtJU&qgDa2V8f?i&3ZU1=_LmZ8mjKO+u)P$sd#ZcHoKNA+|R zR>t=y8+|+LMgCS&3dWQJ`;nHgxR6kZA_T`VboDkiUMoybahiDX3KfB}9G0O#x~}x` zC+`o|-<9)trfuY-r224f#m6;Ijhk-8qRI(;i}!C83_evg+}TC=daW(h?tJjCjg7%Y zpA5oxGH*0cagXb1ZQ2YC_H$=DGDmJUygcuNnQ$~w2#~%KbqzVov0UJ}k};7YzCY*X z(Y1z)TV7zT`qtPe>9%R#R&wg@8NwN*JP#>h*hk828j|I!4JFT%pL)%IcimmeZHue2 z?C2&7^2t}8AK=(sW0~`Mo9r5N?hsl0&O6#|s$Qjas(LQTi`LYsTk~|3xYwquTD@G4 z5ejFr`jeRC5>gpR54kQ~-j9?-GB%INK@NnfFfcHvc3FL-y{X}_CLb-RagPw4E+&4R zRY~=u;r<2O3GC<8Wl`tS8Y?;xSG6cw(WAvX-sFIbY<{DK{A(Yw|J4G3glYQqtn!`! zQzmoa_?4Z%1HMbD>t91+H!pEFFvbZrnAJT=vp>vyBjIz`bBp{`xgEo|a+`x83RCY{<9y|--MB13T)xA|e+R9&Z(#MdLahd;+;j-9VpeN5|HON+}bnV!z%lZ_b z&C_}ZU2~3X5V5V2(amOjDa2H>x37xY56Ij2F!=HIC>uRcWZ?Hq@cfW;+1KDGWWBCc zr6wpW0cmX-KjU-syx4K39ar(vhcC@OOA0P-r@CCRxM)|4Ml^UVa(O%hcZ_}K3Ok5_ z_08-~kjz(MNz(4!g&5}aRWPnIs()C~kZ09O zME1TbX2$13hPlipmhM~1v*gbY`-i)fjgjkC%Kv%g*E^cHnbp01pPbkg4fIy>B&+qoIs>n(sU$nyUsNX(1 z?dsMJ$PPc`WeJhLlFxbWW4`!Yf&D=FUj+n-ac>hNHLl){AftUvbL^rKp5$9tC|y`s zU*Rc32bj6iR54`tqc^3VqNmoq9wXtL?Aj00Ds>V&4(2%|lWbthD6L*0Hnwat5Pn|; zTjg!SPF2bq-7f~Gs#h!zm$n=GqNlbcJH;UeXsSA?YP}h^iQ<(kGWg-mols%r)b4hh z;+jH9EoIB{(LMuWgB5u?s@qCkMLarOwTy}_#hI5)OT1bCfVIw4xyRAL;cbIviJ20x z--EVeAJr$3mx!YnyoDxKd}nBoTT6rbs(%-E8{(?0x`~Xz3LN~)Xfw|COiX->c$-Ld zgyae{)fUv)EB6lVn#q;%5;#8UF1x+yo{?T6{e zocs9zN}`yry~!(FPTZb!>#Jih>r#(e9jsBgqZXGo@{y9AvY8?;ilWpoUiH>1lx9go zoY3w8@#Dc<%^B{=2Y;b2AgqS+7cy!+pG-BFAI#tnxy1U?dRUw~S!90E@c5(+81Dmo zVG-q?P*kbELP)r7V`VO9GNTMR1=_ z->s|j+4S@i6qFoSm>_=5vAOqG!1WCM?H4)y-1HK*=fZC#s=S)-z1`!_`}0*T12#yo zXqFKD5VIP;JXG#8mkf55IEVBqh3nEraIHOB;ho84uH29ur%pgxQx9q!lf8~)c3;{s zcw6kM;QIb($_6Ga%HeUUr^F(ZxVJ~ZkH3P zV%>G7aNnoIWISlREarGL&n-D#IMtP?eW+l3MEpzII4VNib4x7>#+Bprp*0+o3gYH| z2ilR=!~^PoFJEBh*DDORb)Fxwghk8j+Ich{COIH%CQph8PZTRae9P+j;hKlt9=k>+ z_6-YxwjaL(&TdgKag+3|BqI?l?!}KN!9S7ED|PQiypV}Q##jhFIx2jqQhenxdH;z< z?j=!g-_O%djdCYlmwwJIEMyIzUfg_p>TabGL)!3Z?Bc_)yh569uCl7!3ADu_!P6VqOEoh&pcj_;u+?9Z=Aw>VZ zV4f7B^59*}M5$?($n`pE*nl$B$Smd8?H_*x_tQ|`NEY$w0Gk1a7_Q4ZVRi?4)O>Z_kHv(|yIgX2=CZkj(Ly3$g=kKr`j z<>J;pD)snDf^f5=3t_Up+XDqvs{B{jGnpwTPs)ruWp|cho$o6f|I*yi-OPMgwfakR zwXTTkTdfCa^C*V}a;LMThrNA!VZD;GyylHbteB}Dw6GXd>Ul4`Z#JeW1hH1wg*#v) zFGnf~I^Cfojyj>D!d)Ej=u+`M+0kZOR&pNc1dzpZ+*c?msYFsJoIQW`0<8MVZ18)k zZEIe5`&RsHRXNH+sOM=F0KiuWlJO?cD$?Z+i%9GI+9gV{8TH8@x2-fzValeF&2#WW zk#AwxVY<$o;jaaAyR!_)MS@%~HH|V{DJ%5KCgmoZ=iNw8^5fi+D{|H^GaXXh~FY@O_987fbNF(iDpYMz@-LtsN9tKX-M>J{bqgkH#`iA531w973Lcp#$%|4?MBjc7AMYf>XmAE zf4!;rP>OY9lT1_W%-~rdj3y6Tp2Z)C{l)mdbxTeWH`ZJIG&Hx;fG#jVaQ;8{8UXM_dO(HYj}k~{93rg zkT3_MmO(3y+G-&CIs=i{I^hiP0)tA!dB(@H+IA>}Pbmhx`7H!R7iMIWA=iW-{mnBdf(Or9$Q+omI4OYx7( z{#}JI8RxI&W|RFGSVQf@>TY>i);Qp(c`K`__~N1jZ!+oLklbbXfgnwFc0D@U&Yye; zOj3k1V}_Ho)It+jY1A-xCfaXrlp5myifIgZr5zz7t{i*mRAZ_4o>k zJmEpP+I`^LtyIpg?ae&+_#EZsHqtCe&(tQFbY7IxIegItMd*i4W5e=+5c(EWMV2M| ztPnRxa6fNJc2`*PO%jcGZym?y@pMtZX3$*W*R*2MLDlMq$Sy_b5Ybhqdsul819xSP z(;?U-)~5}R6e$@QC-3}@7u`=WO>>eWm@-1!BzY!Ooq!qh43HLoS7U$vJk8#98dGUZ z&2ei0nK(!Nua2G*gr*BP6TQ9Jf|~l{M=JxCw<~O?ZWLCg)m(Hvc6bQ7sCGzd$GKaJ^YAfhBJqsywKY*NH=XDQcK~ zkE&Mf0jPF)1%fkaHuYDPpdcSdk*@=)3E(oqSg}QG41qkKeMhI6N@bgYfiy zkv}d>bR|2fW4U6=R{!Hy`Ocd=tJW!uXmYT; zP+rNn{t_6%qwO~!hd;~5#=JJ^wW^u02j+1qqW!m^Llxxpe?Y|~A=RDaR+rR9|=}=0(x@B6u?Uu;|KoZTJe|5shXtGCa{~)DeYFY9jvSy2J1+K0V zTSu~l^Ymliv~wnv=WO0E;79zo{o@3AYHq=7ZGS*qK z&(^ZZVUQG;>v;7FN3KR}b*O5ipURj+_IO1sg~DabMzzmYij}DYrt+fsuWr%RrO9-2 zSn;*gh->6jbyht`SF7%)I}F}Jr~CI6NC%+UnvA^p9X`&+kWK0(`+Y~Zh`22Pu~Ol7 zqh>L!R`khy!PAi6$$rXzY57_96bS4zSd6DT%F_$jwDNagXfyF3`3LJyFZV1nI}6Ya zaE`9)HGJzcRqNEEv2+}{KKOi0*Q==pI1X=88FGExTE9&V*{LzS0MCBm@b4QgyKa+S z$DdLkD~BcR><+*33-UW?gY#riVI*E0@A6~<1fzHk#;30SZi{kSD8jO?Jggiurgd1o za+un*bd0lpkCT^=k6`kI4-gCzv28qC6ja3L06NqmAhic*8; z`Nt)E+$|@_DBTsuPg?98dgK4vl-G$A*5;nwZAmy81w~%DHkTv$ofbYOCuT%GZUn5xuA>j+9x%!a;Dbjwi$C#%(spJ#15R?y5 z)7b^Se&)2X#{OEcN}^mO)|ZMag-`c3+FuQGi!~)G%oTWdm-LfSrail)F=K~PlVI5z z3J(v5%4L4rE0TW+K~h>6XDGO%?dAO!k>*e=DJo5Wo8Pi;pf{ZiU`!<-F%O9Z)%ls3 zSOm*1D=5f0QK&HZyhRd+Q z(q-8-bNRk6+FAEJ;K2Ih`g`i_2QH6?nk|fmgvpAchM};wK!<-Ew0rF#V>n?Ab&47| zWi~hX{^;f|;q&;vE5U9V6PiK`l6V1|*<}ncI`^6AUMH5?r0HnY$iBYnD)`p1+S}Cy zV#I_8Y#?EPy-ZKxY*u^RZ#6SVdXpiP!KG$7;gSC z&Q^#F;WB&zInH)VS?fHxBq&_$ND)Cke0QArduXO{adx~Zwac>BAwPBgQ4-$!mwg3` z&wG6Tt?qrD(%PdB!8&sNkqpVTts`?>PD{MSV`4(<(f&+3VDWL%g=+ z#4Wb-tr1{wWb~wO-mz5r46616#lRSy1FO;6uMG_fSBd9p);P!yD<96yEizO)ZP&vb zi;Co?{RBW~BX8_&k}K4aHV1ird%i98g9C7?LT+gpNMCXr1=8R=YFR(Od7d1pBcgWE zmCCy)x0zsV90M|!Vfwc$TnBEw{iA2IGnLW>;Eo;4vzx3|RQ1z&l92BIQR)VwC3HU* zG7{xV8IK~=oB(n_k{wQ+f}Ol%^P(@|BjEAXfvehF3GvC%FG{NI79(m`ff|4<9^`jO z<_PJH8ffr(E}a|~(z3h^YJ$&=Er}PyN;L<$akd=1{f>p^kzzmC=&c+ByPzoC3%KStAP@Uxc5 z$F??#RS_5L$nv|r=I;!C(>GI=fV?U$m@b$!u=r@E@KUCUUd-EjER=!XW8)S}NiHb` zQh_Wm6eSO--9J)~)Q!?ac=Ccp7 zl6_L;rDBuAp|dt#{uj;n`kDaVDKpQj#c&j3{5iflci9C8H`}GkA;IJqKsdE}fyVRa zh~$t}`Wd>iG{GyhGbYdbB?xggKQ7~AYEL*q$F+{HH3j1EEW5Jo4KBTpwL(M+y)S^| zf7lxin|Zvjz}i-yheh)A2PwFo%;VFi&shJD$NtiP1gOD+MGK)t4pW3b3Ne|S`hQBE zQWXvBr#+=&->2=Kzf`=_bvuQ^4)~y|Hzrl@qq(&8rbl}lNhtAVILypkJ6FT9^&8oY zief^8ea^;{yK6&4WTiOLGm(=8u(Q(bm1r`lFNN z$JdddXc8Sd$`_$^f&c8bbknq;B%vu_O5d0AB7F%-Wp2lpSk_DeNhr;FbpPW2PJNM? zII5ghiOs%+&P@oL;cjTh)7mIBM>qr6Rg;#`Qmra|rHx2I)g5Aq#HEnzxty%zM3_Lo zhm|&CH8C-Qgy`Dwe?-D2zeU2b<`V9$RLHB_bl(mnf^`n4?n<$arpYME1t46iEHM#~ ztI)kIbCCtK> zeK{KA_;NDY7Q;7=@*bC6S+Zi#{y5m+BpYwpkx~moDbg}tp;vZez1KDlvUg|jCE&qK z>P|aVpev%Ou)R$u?#e16yZGS*K|8g`A`OGSRBX9f!R+?Ers;~26YZ9_eeEX|U|#mN ze5OO~kayJdAQSIxf`v`e?(~_)rPI7iX?kzNW1{zr!e>w?p*3xrdf>zX+r)DF8|4ke z;y})0L7RL4-hmV$BA3NLepBF*WZn~UJw?nFR~qTBrl4&TILOkgDzytxmUQDSh#Flt z29#w$&u(13GzVl5dxE&R^i)eSUu8A+{L2G`0?CbTrilX<8oa~mNXkraDdI*t>;(VO zFgt5UaE+`;3s+q4pzN{>32wG7CXSNrpoEW9V;LuwOs-L&w;5)@rd^m|4u02fZRUy# zxghXYOI)5e_-{U40p6|h>}9+~00J%X+_J{Ws9pzZ$dN5&y>?m54KHYm8_m~Uk0`kx z7H<3@y=`H7j=_seo=K(x=uFk*dq7;Kq!5}yP*I@j!7xCG9-t{p>#Vu^Qma+4PmY~~ z5oCh+Pd=DgaEysQXFzhHS0tan9~Y>6!0ocom+(C@&ySz(c-*8af7j+x@q5m4f3<-R zF8fJhL@ndCO&;H&P!zz~L)5Liz0tzrR_jBCeI#RHdc|E7u9GOnlm`z4ps{Gz3gV9H zAZX57w*|%6>V1O3!r~A|{%UdhK6oJL;co^i(Ym^8gE{IgK*{>_@t;M|L1{O`&)u_c zwYl&$F7T>p58^Vt`dTlDk^ouXdKOV|{ZVuV1-_Sno~|_^`saHGW8t>*q^Rt<4%Nq9 z3%b}9P(KW)2s_`A$Z{veT;;RpQ>+I3p(1{mKi_vO!LYe=Sp~A;{cYX3rwswVvD!?p zeE)7io5$B{bk$A=INZ&<*Z%SKOO`m&+M{0;@A68O!IVtAy^Mqelb_(dfo~=k5FptC zpui~URIx4(J2c5-YZuTO_WegN9Llq{>}-~h{!*rlU05QHva9F+Bp=7h6j5%R(N0Gk z2f;+duz?ty=g+G`u@-iV`tTh!r5FlM@;?O|qql9k#uin4MP2k3GvIL0i7XsjyD^2_ z!q8qS*zLMhEW|nMulCddpayuEmV&Dax)8rZyK|LfVfJZ*=LM7yuY)y0YbDHSnXH4s z3JFN*kdtB+VW5n3pYqwBVcCl^X$QVb?6A|x)ymbu;cQ4~WRape>=*mv&keW)3GT_d z!>#d1^#attWfY;7!qje!2eww#<-#D>zgu`JD(3caPlPq!is+ka`$yOarf9wyH^(O4 ze5M^ZQHmltmUDFHKIOg(&tf&;o6ii%A+cddsXZ1++2X=rBl6S%?h zNxW0DJgUJfUi1%7LyG_zCWi2x0bAg!5-lWWYG}&OqXRhTygjB4h#D!cO@8rLBe5SO zwV9Jlq2=AUD`(|8zWmeXk0WqTkAA<7`yLr%Vp4XV6`Izhl^mK0uT>lrK6$0scOVNx z2Einx<)#3_jcksz^_QPqj<#xR705Ct4!pNKQnn3qym*&_deoqWlya{SpGCmkmc?c`?jQi$Hb8OE?XEBa^twyU_??&wGI% zK>*B~5><+uHK+eN0O2b@${UyC?R#y@n5{DX9_|3qf6p%?Zi+#~8n!yS_A&E?I0~h) z_8^(2G4a-E8ILE-C}ELHso!s5ZBK&CQdM~{iGbkDZTWlB8X6KDyxnbYseLXJmoGyb z?$TI>^9-ya+;lX>+zt79H6Zsi86Mc31yJ3vyHz6}nZY-g5sL7Ngo#M*@Y%hQ2|(^g{W&N|!GsTC}x>3B~+6Amu&;m6cqNKK{Sm zyBB3u?=Z76Fw;@OWju8A9Rb0Pq>EInB`=!maP8$W3O5DK-4cW6?`5vwQ7WU!5fsFu zxs1FUf38%zD>0rWAUJC(#ac;w*{jj-eym`R^L+NKAo9=ezE>;HSv;p6!}ZQ@MuqGg z+Y!$R0;#Y>f3-LCv}vsfXep^1N)tCFkM~{hDz=YnOjD!gjPM(+pcOUhymgYm|LQ&= z`qN%=IAi*Q{RMNyJJ4bUuPVjAII=Sv!q~GESH=tz2;oO>(4LBVzLD~@3U zO#ygM>(9^L>uj*h)0+D5chlryGldud!Oy@Fe>HnnW-gaDa?=M@_NfcsE-A)ou8*_- z9wsBBh?>NwR`M8kAG7U-V@j2I0)jUTxHAZ1Z8f!iU)>yfh1hC~w+r*P=pX!!!C(o~ zO0V=QW%XPgK1_;~`N}+I)Dt^X_oNLS+-eehjer2B0@UV<-Bx2)UbrOSf$y`pDUUzn zzNBh#;T*s1xI<&WSo!jYx_F1(V_wc7MkN{o0upIoB8AU0O&O)e!-x|KAqx@C@g+?+ zGY&T*E7xL=L~KP|KYr8W(6+8n?w>lIe4sfXpZMRuOsl%nTVDGu@!L+um7CdMaWk$h zlr!E=;ffC$4*X(uWnwr92qq|q&mfrkdiKWX^Q|U)4;vcaAB!%|Yw7&O`PErJF-Ora zLm7+z>Lyp8Q~UilWh_U(OLVkfcy^>nPRT=v7^3NE)hM2wAox^}_K6VPH1l4kzYsI) z|8QKXf2U=V-2F!8Kfgm?+uU;Zxzax$HZZTG^uX&L{@w0i<)a(DHwmIhNNA8_t(|)f z9ok3Shqz;TvMEksL6d(TTssI4<{tTHth$vViX9Dd_;(APr7nwILE)J}FHChgDsJ*a zJE>E)0x40EoSm}&?1u9v2@vRsc7XB!zEY$vB2WDL#^3CJ-u%xT5D@&&9Q;4d!GWKs zD+m7c@U%{OUhyKiK_Kw__t_dbqk#YR!y5=f_{0A-aLe%e|MQQRM{Lc{QC?AC#6MDA MM)_Xhorll=7j}I}T>t<8 literal 0 HcmV?d00001 diff --git a/docs/source/conf.py b/docs/source/conf.py index fe398a4..d439e8c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -14,11 +14,9 @@ project = 'PQuantML' copyright = '2025, Roope Niemi' author = 'Roope Niemi, Anastasiia Petrovych' -release = "1.0.0" +release = "0.0.6" version = release -# -- General configuration --------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration myst_enable_extensions = [ "amsmath", @@ -58,8 +56,6 @@ 'conf_py_path': '/docs/', # Path in the checkout to the docs root } -# -- Options for HTML output ------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output html_theme = "sphinx_rtd_theme" html_static_path = ['_static'] diff --git a/docs/source/faq.md b/docs/source/faq.md index c8637a9..5af24cf 100644 --- a/docs/source/faq.md +++ b/docs/source/faq.md @@ -1,16 +1,23 @@ # FAQs ## What models formats does PQuantML currently support? -PQuantML primarily supports PyTorch and TensorFlow/Keras models and supports both direct construction and automatic layer replacement using `add_compression_layers(...)`. +PQuantML primarily supports PyTorch and TensorFlow/Keras models and supports both direct construction and automatic layer replacement using `add_compression_layers(...)` method. -## What are requirements to use PQuantML? -Install PyTorch with the correct CUDA version that matches your system and other frameworks, like TensorFlow. This prevents version mismatches and GPU compatibility issues. + +## What are the requirements for using PQuantML? +PQuantML supports two backends. If you are using the PyTorch backend, make sure to install a version of PyTorch built for the CUDA version installed on your system. If you are also using frameworks such as TensorFlow, ensure that all frameworks are compatible with the same CUDA version to avoid version conflicts and GPU compatibility issues. An example to install PyTorch with CUDA 13.0: ```python pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cu130 ``` + +## Can I export models to ONNX? + +Yes. PQuantML supports exporting compatible models to the **ONNX** format, making it easy to deploy quantized models across different inference runtimes and hardware platforms. + + ## Can I use MLflow locally? Yes. @@ -21,8 +28,11 @@ PQuantML integrates with MLflow for experiment tracking and model logging and lo ```python mlflow ui --host 0.0.0.0 --port 5000 ``` +By default, the MLflow UI is available at `http://localhost:5000`. + +### Use a local or remote Optuna database: +PQuantML also supports storing Optuna studies in either a local SQLite database or a remote database server. -### Use a local or remote database for Optuna tuning: ```python from pquant.core.finetuning import TuningTask tuner = TuningTask(config) diff --git a/docs/source/getting_started.md b/docs/source/getting_started.md index c5c99d6..0ff3ba1 100644 --- a/docs/source/getting_started.md +++ b/docs/source/getting_started.md @@ -1,19 +1,20 @@ # Quick User Guide ```{note} -This section provides an overview of how to use the PQuantML library: defining models with pruning and quantization, running fine-tuning, and optionally converting the final model to hls4ml. +This section provides an overview of how to use the PQuantML library: defining models with pruning and quantization, running hyperparameters optimization, and optionally converting the final model to hls4ml. ``` -## Model definition & training - -To compress a model with PQuantML, all layers must be replaced with their PQuantML equivalents. For example, replace `Dense` by `PQDense`, `ReLU` by `PQActivation`, etc. +## Model definition & training +To enable pruning and quantization, a model must use PQuantML layers. This can be done in one of two ways: +- Direct layer definition, by building the model with PQuantML layers such as PQDense and PQActivation. +- Automatic layer replacement, by converting an existing PyTorch model using add_compression_layers(...). -Model compression behaviour such as pruning strength, quantization bit-widths, training parameters, etc. is controlled through the configuration object, which is a Pydantic model. +Model compression behaviour such as pruning strength, quantization bit-widths, training parameters, etc. is controlled through the configuration object, which is a Pydantic model to provide an automatic type checking. -### Load a default configuration -``` python +### Load the default DST configuration +```python from pquant import dst_config # Upload a default DST config @@ -57,7 +58,7 @@ def build_model(config): return Model(config) ``` - +This approach is recommended when developing a new architecture from scratch. ### Layer-replacement usage ```python @@ -86,8 +87,10 @@ def build_model(): model = add_compression_layers(model, config) ``` -### Fine-Tuning with PQuantML -PQuantML provides an automated fine-tuning and hyperparameter-optimization workflow through the `TuningTask API`. This allows you to search for optimal pruning, quantization, and training parameters using your own training, validation, and objective functions. +If you already have a model, it can be converted automatically by replacing supported layers with their PQuantML equivalents. + +### Hyperparameters optimization with PQuantML +PQuantML provides an automated hyperparameter-optimization workflow through the TuningTask API. This allows you to search for optimal pruning, quantization, and training parameters using your own training, validation, and objective functions. ```python from pquant.core.finetuning import TuningTask, TuningConfig @@ -114,7 +117,7 @@ tuner.set_optimizer_function(get_optimizer) tuner.set_scheduler_function(get_scheduler) ``` -To run optimization: +Run optimization: ```python device = "cuda" if torch.cuda.is_available() else "cpu" model = model.to(device) @@ -124,10 +127,10 @@ best_params = tuner.run_optimization(model, testloader=..., loss_func=...) ``` + ```{note} -`tuner.run_optimization()` automatically runs multiple compression–fine-tuning cycles, evaluates each trial using your objective function, and returns the best hyperparameters. +`tuner.run_optimization()` automatically runs multiple compression cycles, evaluates each trial using your objective function, and returns the best hyperparameter configuration. ``` - All other training code remains unchanged. ### Train a model @@ -153,12 +156,13 @@ trained_model = train_model(model = model, optimizer = optimizer, scheduler=scheduler ) -``` +``` ### Using different quantization settings per layer ```{note} -For FITCompress, HGQ, or architectures, where activations require different quantization bit-widths, each activation layer must be instantiated separately. +If different activation layers require different quantization settings (for example when using FITCompress or HGQ), instantiate each `PQActivation` layer separately instead of reusing a single activation module. ``` + ```python def build_model(config): class Model(torch.nn.Module): diff --git a/docs/source/index.rst b/docs/source/index.rst index e697789..ae6d779 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,7 +1,7 @@ .. PQuantMLdocumentation master file, created by sphinx-quickstart on Mon Dec 8 16:28:11 2023. You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. + contain the root toctree directive. =========================== PQuantML @@ -14,19 +14,23 @@ PQuantML .. image:: https://badge.fury.io/py/hgq.svg :target: https://pypi.org/project/pquant-ml/ -Welcome to the official documentation for **PQuantML**, a hardware-aware model compression framework supporting: +Welcome to the documentation for PQuantML, a hardware-aware model compression framework supporting: - Joint pruning + quantization - Layer-wise precision configuration - Flexible training pipelines - PyTorch and TensorFlow backends +- Knowledge distillation +- HGQ library intergration - Integration with hardware-friendly toolchains (e.g., hls4ml) PQuantML enables efficient deployment of compact neural networks on resource-constrained hardware such as FPGAs and embedded accelerators. +The paper describing the framework is available at: `PQuantML: A Tool for End-to-End Hardware-aware Model Compression `_. + .. rst-class:: light -.. image:: _static/overview_pquant.png +.. image:: _static/overview_pquant_updated.png :alt: PQuantML-overview :width: 100% :align: center @@ -37,12 +41,12 @@ PQuantML enables efficient deployment of compact neural networks on resource-con Key Features ------------ -- **Joint Quantization + Pruning:** Combine bit-width reduction with structured pruning. -- **Flexible Precision Control:** Per-layer and mixed-precision configuration. -- **Hardware-Aware Objective:** Include resource constraints (DSP, LUT, BRAM) in training. -- **Simple API:** Configure compression through a single YAML or Python object. -- **PyTorch Integration:** Works with custom training/validation loops. -- **Export Support:** Model conversion towards hardware toolchains. +- Joint Quantization + Pruning: Combine bit-width reduction with structured pruning. +- Flexible Precision Control: Per-layer and mixed-precision configuration. +- Hardware-Aware Objective: Include resource constraints (DSP, LUT, BRAM) in training. +- Simple API: Configure compression through a single YAML or Python object. +- PyTorch Integration: Works with custom training/validation loops. +- Export Support: Model conversion towards hardware toolchains. Contents ========================= @@ -61,4 +65,4 @@ Indices and tables ================== * :ref:`genindex` -* :ref:`search` +* :ref:`search` \ No newline at end of file diff --git a/docs/source/install.md b/docs/source/install.md index e9ba183..8fed3e6 100644 --- a/docs/source/install.md +++ b/docs/source/install.md @@ -1,8 +1,10 @@ # Installation +Installation via pip: `pip install pquant-ml`. -Use `pip install pquant-ml` to install the latest version from PyPI. You will need an environment with `python>=3.10,<=3.12` installed. +With TensorFlow backend: `pip install pquant-ml[tensorflow]`. +With PyTorch backend: `pip install pquant-ml[torch]`. ```{warning} -PQuantML v1.0 requires `tensorflow>=2.17`, `mlflow>=2.0,<3.0`, and `python>=3.10,<=3.12`. +PQuantML v0.0.6 requires `tensorflow>=2.17`, `mlflow>=2.0,<3.0`, and `python>=3.10,<=3.12`. ``` diff --git a/docs/source/reference.md b/docs/source/reference.md index 10802c7..e74cc74 100644 --- a/docs/source/reference.md +++ b/docs/source/reference.md @@ -2,19 +2,20 @@ ## Config file -The most important part of the library is a user-defined config yaml file. It has five separate sections: training, pruning, quantization, finetuning, and fitcompress section, `currently maintained by TensorFlow only`, parameters. By default, the parameters in the config are the following: +The most important part of the library is a user-defined `config.yaml` file. It has five separate sections: **training, pruning, quantization, hpo, and fitcompress**. By default, the parameters in the config are the following: ### Training parameters -The following table outlines the primary parameters used to configure the training process: +The following table outlines the description of default parameters used to configure the training process: | **Field** | **Type** | **Default** | **Description** | |------------------------|---------------------------------------------|---------------|------------------------------------------------------------| -| `epochs` | int | `200` | Total number of training epochs. | -| `fine_tuning_epochs` | int | `0` | Additional epochs for fine-tuning. | -| `pretraining_epochs` | int | `50` | Pretraining / warm-up epochs. | -| `rewind` | str | `"never"` | Weight rewinding policy. | -| `rounds` | int | `1` | Number of prune–fine-tune cycles. | -| `save_weights_epoch` | int | `-1` | Save checkpoint at this epoch (`-1` disables). | +| `epochs` | int | `200` | Number of epochs during the main training stage. | +| `fine_tuning_epochs` | int | `0` | Number of epochs during the fine-tuning stage. | +| `pretraining_epochs` | int | `50` | Number of epochs during the pretraining stage. | +| `rewind` | str | `"never"` | When to rewind the weights: never, every-round, or posttraining-stage. | +| `rounds` | int | `1` | Number of pruning/quantization rounds during training. | +| `save_weights_epoch` | int | `-1` | Epoch at which weights are saved for rewinding during the first round. | +| `pruning_first` | bool | `False` | Whether to prune before quantization. If false, pruning occurs after quantization.| ```{note} If you require additional parameters for the training or optimization loops, please define them directly in the config.yaml file. @@ -33,10 +34,11 @@ If you require additional parameters for the training or optimization loops, ple | `quantize_input` | bool | `true` | Whether inputs to layers are quantized by default. | | `quantize_output` | bool | `true` | Whether outputs of layers are quantized by default. | | `enable_quantization` | bool | `true` | Global switch to enable or disable quantization. | +| `granularity` | str | `"per_tensor"` | Whether bitwidths are shared across the whole tensor, per-channel, or per-weight. | | `hgq_gamma` | float | `0.0` | HGQ regularization coefficient for bitwidth stability. | | `hgq_beta` | float | `0.0` | HGQ loss coefficient scaling EBOPs. | | `layer_specific` | dict | `{}` | Dictionary for per-layer quantization overrides. | -| `use_hgq` | bool | `false` | Enable or disable High Granularity Quantization (HGQ). | +| `use_high_granularity_quantization` | bool | `false` | Enable or disable High Granularity Quantization (HGQ). | | `use_real_tanh` | bool | `false` | Use a real `tanh` instead of hard/approximate `tanh`. | | `overflow_mode_data` | str | `"SAT"` | Overflow handling mode for input and output quantizers(`SAT`, `SAT_SYM`, `WRAP`, `WRAP_SM`). | | `overflow_mode_parameters` | str | `"SAT"` | Overflow handling mode for weight and biases quantizers(`SAT`, `SAT_SYM`, `WRAP`, `WRAP_SM`). | @@ -44,7 +46,7 @@ If you require additional parameters for the training or optimization loops, ple | `use_relu_multiplier` | bool | `true` | Enable a learned bit-shift multiplier inside ReLU layers. | -### Fine-tuning parameters +### Hyperparameters optimization parameters | **Field** | **Type** | **Default** | **Description** | |-------------------------|--------------------------|--------------------|-----------------------------------------------| @@ -163,24 +165,24 @@ There are more details about every pruning method: | `t_start_collecting_batch` | int | `50` | Steps to skip before collecting statistics. | #### MDMM Pruning - -| **Field** | **Type** | **Default** | **Description** | -|--------------------|-----------------------|----------------------------|--------------------------------------------------------------| -| `pruning_method` | str | `mdmm` | Selects this pruning schema. | -| `constraint_type` | ConstraintType | `"Equality"` | Constraint form: equality / ≀ / β‰₯. | -| `target_value` | float | `0.0` | Target value for the chosen metric. | -| `metric_type` | MetricType | `"UnstructuredSparsity"` | Specifies which metric is constrained. | -| `target_sparsity` | float | `0.9` | Target sparsity when constraining sparsity. | -| `rf` | int | `1` | Regularization / frequency parameter. | -| `epsilon` | float | `1.0e-03` | Feasibility tolerance. | -| `scale` | float | `10.0` | Penalty scaling for constraint violation. | -| `damping` | float | `1.0` | Damping term for numerical stability. | -| `use_grad` | bool | `false` | Use gradient information during updates. | -| `l0_mode` | `"coarse"` \| `"smooth"` | `"coarse"` | L0 approximation mode. | -| `scale_mode` | `"mean"` \| `"sum"` | `"mean"` | Aggregation mode for penalties. | - - -Optionally, there is also FITCompress method implemented for PyTorch: +| **Field** | **Type** | **Default** | **Description** | +|--------------------|-----------------------|----------------------------|---------------------------------------------------------------------------------| +| `pruning_method` | str | `"mdmm"` | Selects this pruning schema. | +| `constraint_type` | ConstraintType | `"Equality"` | Constraint form: `Equality`, `LessThanOrEqual`, or `GreaterThanOrEqual`. | +| `target_value` | float | `0.0` | Target value for the chosen metric (used for the general constraint). | +| `metric_type` | MetricType | `"UnstructuredSparsity"` | Metric being constrained: `UnstructuredSparsity` or `StructuredSparsity`. | +| `target_sparsity` | float | `0.9` | Target sparsity when constraining sparsity. | +| `rf` | int | `1` | Regularization / frequency parameter. | +| `epsilon` | float | `1.0e-03` | Feasibility tolerance. | +| `scale` | float | `10.0` | Penalty scaling for constraint violation. | +| `damping` | float | `1.0` | Damping term for numerical stability. | +| `use_grad` | bool | `false` | Use gradient information during updates. | +| `l0_mode` | `"coarse"` \| `"smooth"` | `"coarse"` | L0 approximation mode. | +| `scale_mode` | `"mean"` \| `"sum"` | `"mean"` | Aggregation mode for penalties. | +| `constraint_lr` | float | `1.0e-03` | Learning rate for the Lagrange multiplier (dual variable). | + + +Optionally, there is also FITCompress method implemented for PyTorch-only: ### FitCompress method | **Field** | **Type** | **Default** | **Description** | |---------------------------|----------|-------------|---------------------------------------------------------------------------------| @@ -201,9 +203,19 @@ Optionally, there is also FITCompress method implemented for PyTorch: - `PQAvgPool*D`: Average pooling layers. - `PQBatchNorm*D`: BatchNorm layers. - `PQDense`: Linear layer. -- `PQActivation`: Activation layers (ReLU, Tanh) +- `PQActivation`: Activation layers (ReLU, Tanh, Leaky Relu, Gelu, Hard Tanh, or a user-provided activation function (Torch only) ). +- `MultiHeadAttention`: Multi-head attention layer. +- `LayerNorm`: Layer normalization layer (Currently Torch only). ```{note} -Currently, PQuantML supports two quantization modes: layer-wise fixed-point quantization, where each tensor uses a single -bit-width configuration, and High-Granularity Quantization (HGQ). -``` +PQuantML supports two quantization modes, each with several granularity options: + +**Fixed-point quantization** (for weights): +- per-weight +- per-channel +- per-tensor + +**HGQ (High Granularity Quantization)**: +- per-weight (learned bit-widths per weight) +- per-tensor (learned bit-widths per tensor) +``` \ No newline at end of file diff --git a/docs/source/status.md b/docs/source/status.md index 8dd291f..276021b 100644 --- a/docs/source/status.md +++ b/docs/source/status.md @@ -1,15 +1,23 @@ # PQuantML Status -This page tracks the development status of PQuantML features +This page tracks the development status of PQuantML features. -## Release: v1.0.0 +## Release: v0.0.6 | Feature | Status | Notes | |---------------------------------|-----------------|-------| -| Compression pipeline | βœ… Complete | Included in v1.0.0 | +| Compression pipeline | βœ… Complete | Included in v0.0.6 | | Pruning methods (7 variants) | βœ… Complete | All documented | | Quantization (fixed-point) | βœ… Complete | Supports per-layer overrides | | HGQ support |βœ… Complete | Supports HGQ quantization | -| hls4ml integration | βœ… Complete | Works in v1.0.0 | -| FITCompress | 🚧 Partially implemented | Works through PyTorch only | -| Documentation | 🚧 Improving | Expanded daily | +| hls4ml integration | βœ… Complete | Works in v0.0.6 | +| FITCompress | βœ… Complete | Supported in PyTorch only | +| Model fit support | βœ… Complete | Works in v0.0.6 | +| Alkaid converter support | ⏳ Coming in v0.0.7 | Implemented in dev | +| Onnx converter support | ⏳ Coming in v0.0.7 | Implemented in dev | +| Knowledge distillation | ⏳ Coming in v0.0.7 | Implemented in dev | +| Implementation of HGQ and pruning layers in Torch | ⏳ Coming in v0.0.7 | Implemented in dev | +| Additional test coverage for pruning methods | ⏳ Will be in the next release | Implemented in dev | +| CI/CD pipeline | 🚧 Work in progress | Due to the next release | +| MDMM pruning algorithm metrics extension | 🚧 Work in progress | Due to the next release | +| Documentation | 🚧 Improving | Expanded monthly | From 1618106af50233ce39ae6267aa2fd68ac41b4296 Mon Sep 17 00:00:00 2001 From: Anastasiia Petrovych Date: Wed, 29 Jul 2026 16:33:53 +0200 Subject: [PATCH 16/22] Modified readme file (#62) * Modified README file * Added pruning methods overview * Modified the formatting error --- README.md | 141 ++++++++++++------ .../_static/pruning_methods_overview.png | Bin 0 -> 202279 bytes 2 files changed, 99 insertions(+), 42 deletions(-) create mode 100644 docs/source/_static/pruning_methods_overview.png diff --git a/README.md b/README.md index f10cb68..fc3a782 100644 --- a/README.md +++ b/README.md @@ -1,61 +1,118 @@ -![alt text](docs/source/_static/pquant_white_font.png) +

+ PQuantML logo +

-## Prune and Quantize ML models -PQuant is a library for training compressed machine learning models, developed at CERN as part of the [Next Generation Triggers](https://nextgentriggers.web.cern.ch/t13/) project. +# PQuantML -Installation via pip: ```pip install pquant-ml```. +**PQuantML** is an end-to-end library for training compressed machine learning models, developed at CERN as part of the [Next Generation Triggers](https://nextgentriggers.web.cern.ch/t13/) project. -With TensorFlow ```pip install pquant-ml[tensorflow]```. +It supports: -With PyTorch ```pip install pquant-ml[torch]```. +- Joint pruning + quantization +- Layer-wise precision configuration +- Flexible training pipelines +- PyTorch and TensorFlow backends +- Knowledge distillation +- HGQ library integration +- Integration with hardware-friendly toolchains (e.g., [hls4ml](https://fastmachinelearning.org/hls4ml/)) -PQuant replaces the layers and activations it finds with a Compressed (in the case of layers) or Quantized (in the case of activations) variant. These automatically handle the quantization of the weights, biases and activations, and the pruning of the weights. -Both PyTorch and TensorFlow models are supported. +PQuantML enables efficient deployment of compact neural networks on resource-constrained hardware such as FPGAs and embedded accelerators. -### Layers that can be compressed +

+ PQuantML overview +

-* **PQConv*D**: Convolutional layers -* **PQAvgPool*D**: Average pooling layers -* **PQBatchNorm*D**: BatchNorm layers -* **PQDense**: Linear layer -* **PQActivation**: Activation layers (ReLU, Tanh) +## Installation -The various pruning methods have different training steps, such as a pre-training step and fine-tuning step. PQuant provides a training function, where the user provides the functions to train and validate an epoch, and PQuant handles the training while triggering the different training steps. +Install the base package via pip: +```bash +pip install pquant-ml +``` -![alt text](docs/source/_static/overview_pquant.png) +Install with a specific backend: +```bash +pip install "pquant-ml[tensorflow]" # TensorFlow backend +pip install "pquant-ml[torch]" # PyTorch backend +``` +## Supported layers -### Example -Example notebook can be found [here](https://github.com/cern-nextgen/PQuantML/tree/main/examples). It handles the - 1. Creation of a torch model and data loaders. - 2. Creation of the training and validation functions. - 3. Loading a default pruning configuration of a pruning method. - 4. Using the configuration, the model, and the training and validation functions, call the training function of PQuant to train and compress the model. - 5. Creating a custom quantization and pruning configuration for a given model (disable pruning for some layers, different quantization bitwidths for different layers). - 6. Direct layers usage and layers replacement approaches. - 7. Usage of fine-tuning platform. +| Layer | Description | +| --- | --- | +| `PQConv*D` | Convolutional layers | +| `PQAvgPool*D` | Average pooling layers | +| `PQBatchNorm*D` | Batch normalization layers | +| `PQDense` | Linear (fully connected) layer | +| `PQActivation` | Activation layers: ReLU, Tanh, Leaky ReLU, GELU, Hard Tanh, or a user-provided activation function (Torch only) | +| `MultiHeadAttention` | Multi-head attention layer | +| `LayerNorm` | Layer normalization layer (currently Torch only) | -### Pruning methods -A description of the pruning methods and their hyperparameters can be found [here](docs/pruning_methods.md). +## Training -### Quantization parameters -A description of the quantization parameters can be found [here](docs/quantization_parameters.md). +Different pruning methods involve different training stages, such as pre-training and fine-tuning. PQuantML provides a generic training function: you supply your own training and validation functions along with the number of epochs, and PQuant handles the training loop while automatically triggering the appropriate stages for the chosen pruning method. +

+ Pruning methods overview. +

+## Quantization -For detailed documentation check this page: [PQuantML documentation](https://pquantml.readthedocs.io/en/latest/) +PQuantML supports two quantization modes, each with several granularity options. +**Fixed-point quantization** (for weights): +- per-weight +- per-channel +- per-tensor -### Authors - - Roope Niemi (CERN) - - Anastasiia Petrovych (CERN) - - Arghya Das (Purdue University) - - Enrico Lupi (CERN) - - Chang Sun (Caltech) - - Dimitrios Danopoulos (CERN) - - Marlon Joshua Helbing - - Mia Liu (Purdue University) - - Michael Kagan (SLAC National Accelerator Laboratory) - - Vladimir Loncar (CERN) - - Maurizio Pierini (CERN) +**HGQ (High Granularity Quantization):** +- per-weight +- per-tensor + +## Example + +Example notebooks are available in the [`examples/`](https://github.com/cern-nextgen/PQuantML/tree/main/examples) directory. It shows how to: + +1. Create a Torch model and data loaders. +2. Create the training and validation functions. +3. Load a default configuration for a pruning method. +4. Train and compress the model by passing the configuration, model, and training/validation functions to PQuant's training function. +5. Build a custom quantization and pruning configuration for a given model (e.g. disabling pruning for some layers, or using different quantization bit-widths per layer). +6. Use the direct-layer and layer-replacement approaches. +7. Use the HPO platform. + +## Documentation + +Full documentation is available at [pquantml.readthedocs.io](https://pquantml.readthedocs.io/en/latest/). + +## Citation + +The framework is described in **PQuantML: A Tool for End-to-End Hardware-aware Model Compression** ([arXiv:2603.26595](https://arxiv.org/abs/2603.26595)). + +If you use PQuantML in your work, please cite: + +```bibtex +@article{niemi2026pquantml, + title = {PQuantML: A Tool for End-to-End Hardware-aware Model Compression}, + author = {Niemi, Roope and Petrovych, Anastasiia and Das, Arghya and + Lupi, Enrico and Sun, Chang and Danopoulos, Dimitrios and + Helbing, Marlon Joshua and Liu, Mia and Kagan, Michael and + Loncar, Vladimir and Pierini, Maurizio}, + journal = {arXiv preprint arXiv:2603.26595}, + year = {2026} +} +``` + +## Authors + +- Roope Niemi (CERN) +- Anastasiia Petrovych (CERN) +- Arghya Das (Purdue University) +- Enrico Lupi (CERN) +- Chang Sun (Caltech) +- Dimitrios Danopoulos (CERN) +- Marlon Joshua Helbing +- Mia Liu (Purdue University) +- Michael Kagan (SLAC National Accelerator Laboratory) +- Vladimir Loncar (CERN) +- Maurizio Pierini (CERN) \ No newline at end of file diff --git a/docs/source/_static/pruning_methods_overview.png b/docs/source/_static/pruning_methods_overview.png new file mode 100644 index 0000000000000000000000000000000000000000..9539e707e93d6bcf828136c277567158faf92e37 GIT binary patch literal 202279 zcmeFZWmFwq^Dl@6cL?t8?gR-TxVvj`cXxLP!QF$q1$TFMcXx+5kmvc|_r7=5tTmr! zKAd$6Bq~-@D2QaE)(S6t6=<@ zp#S~`>3knJggz$?1jG*{D)?2w8TdHulMbpddfym5x)U)S2bfdBrkn=VID&DZ%;Mza zWvzAu+M!gM9CQz5PbkW=fH!1!M1$Mt+0&S%tv^XFtp)*v6_k@(HM7;~^R1=fYIBlm zR)-Ga_*bxCAS411L2pPzU?Tqa0|p2XB0HI%_9Xv#@?!)8$dC5%yZ<%vpWCVl`N6{% zmZ{`^{SS?Qt}yBBbAQnM=a>j^;JtuI0NQJA`I%DezaNdsaE<#PBR}|RH;0S{)@Q2^ zMJ4@jeqb>k;s0g&-`jN9!1}!zQ`EnQBL6d;f3NU^!+JpcA9KY9Oet}sl<0fNe@TG= zG_V!)Kcwa z-@IWlsv8w^DgJkb5WI`s>Hin||JTI6?n#`QFe_ z5t)9R-@iaaZVJss2@(FaWNU0NlKPmypMU+SosYcz&cM$0GDGWrvPV3|5t`d)Jd@E_ zxBmXg`4hy0<1r4Cv9TAuG<%Z7?N|?6LZU_+jsYrkz{vMJ!`z@;f^q?pLF#_$$)jtc zkOs^`p`s7E!}0I_;C3Wp1Mr98P@upyYAUKQ7;!2=u^4`em=Fgp&E1>hSaQ04&$XFK z{o;TzUPjkzhJen_a2nuRwb)fLLCej*g8Apv@foZ89Y z%%XrOu?|ei4?Ya)x>Yz2nB1}f$o|A237TACr=0GB6Mq!m$~Q4FF{;5qdhN(&TT(J6 zs*RjDZCB@p+tU?LyQ<_ixUMD*XqYf9SI>o;;btV_pf7Kov3Tj0dX`+vnubf})y$CR zQFraE>bRne?~d*LfCNy<^C_l@1mHjw84(%|yya_otZ1HJ(CyP$s)<_ZXy%--kfvty zep^V$g6cv5#2vobT2^UkL~W&a?%D206U0buVO*l{ zG|^is#xqMNjju(aDD!$qo59_!68ZMr{?u|G3DF)wV4-aOWK>IwT}(I>|D9p-!r>mQ z0d6M|?yfVi&Yxu}UY1dKL6xz5uSULG%6oZX6USE=i>s}n!6d|x%>d1^+b?WJ((Ear zkGvJa=#qv)M+-A$a#InJh~!70!psVR?$dG(+#8#GbS?FFIf+0bK#vARib0=x77+ma7S3!?g zdLk~y`p}V*hQ{-O(VD4)K!dhSjh#`aa>=ut&Nu}4J&o2gDt;dYbP-y@ujgi04-?fM ze;Z=8I@tS{yOX_94b)&Ty!6wkFHEHlL;Sq9e&x|hDk{yp32X9l%um-RyoT2maM-UL zUc3@yXk%WcQhTGg(V;oAb+30P8|dZx3REKSknpLj`c-Xh2=pjO*2rj%2VVi&fEQ%;sNfHT{+*l>V?;LhLGHD@stW#)4na@dFrIy`+StCBYUptsjU*Q4VKKaxVZ3VIk70Bx=d?!e>?HPlv#`^$ zq*MBa3D!BWB|MzW*|^Kq^4IX%*8o|;?|uM6`?>D*DeD`=`d zID5&GD=Sr*_*`AuANz;(v@tr3h7x@bTVOWO`P!{=((*6dRS7VRt_ZNJ)OHx(;(_C77P%@){TFP zbaU68-k(>0W-gH4oa$tuqM}LWy0{opF2fXLDixwe3h{ce>`K9J-ks=_w?bUEB*G=S za&n*A#igfl)l@9hWsCUwn|1$f(ZhM_5AE9PePd7%cBiv?7`jyX-{nUp1F$mOB*{R& zFKX@AV2Mbb&s0RE<1%t;F>y%|QQyDkZ6pw_dhHQk+nyYk3Ps{eUB=`ZEohF9;AxOfE1UKaNB4!&@OqjUiV%NCW{fH`OGE|^M#Yq>2)jY__(A% z0()y+CpIzF^Yl4x6M08D{=5)fUr!zcfZh2zW}e#JR#B}M-szMP zFHOQwUj&|(Mi#HlX1J7<@kA$au6{9IgBDMBv!yiL;o&u^3kbjg>juF1F?#yj-x4vcy8|(}Tn5B0Pz{9~+4X*eoHku8yq&3YLquTZuBR)Ym=!`R*a>+mkvKjqMc-@5A{~ty%roP23M<2e&XOxuXB8T;2@eFKUJ;-Jj#3i@B)dX$Ny~!Q3r(0Z;1u7AmI^WNg+cSG( z=`9Q%k0jdK^!p_<*u1a%M~d2|l+?xHp^+}X8^E~X+$#@rj-NL#a`FufcW+MCUT)a6 zYU=8K1}~7XBtknt+7>U|s6|NW>hl<_hWG7_T07Z=gh;A9UUC;m2MKFONC!&?0{hjT zSt}9>lay}-x5EQldy({!JDn_^0!Z!8)RWVo{Gz6$yxGH_+u8m#*pux7W5{!PZ$E4| zG(`Vsw{)tg{q{4-?EX0xf3e1d*;)p_qKAb2zN%p^1(oHo8z^=x;k(e6v{fZS_2)_X zY9r~G!CWu%rI+0!{9g(b0f`XutLdBZZkM;YBf-vGor<#kc7q>s@G`R%qzkZ$Q z!!K8RS>ehG_+-fw+Pzo*SWwH=QW6k!^h}PA$$U%5TmH4_R#i>)?qKSQENvh3-O~Zt znJ@ne+b-G!MdIJR<7qTdDgg$wfV5U(in2Q|&ExU|MFVd)vdLLGuD0_0Y+lVlYuq|Hb=p`8+_t_yg7PU&i=&9OM4!kFgRFL6dJmhuDyP8Qlw-O8(X%w zhfZW?4oRCJN>ZT+3j(!dByJo)n@ziboCc0ims7#NR`=~Fh&;m4!NxbeLS=V2jW8}m zd~mSZ?y>?B@)H$3Jv~+D&m_5KG&D4UZ(3M`P&u7F3!bmO!oAWM2ZzTPILBxE#!i>p zN&Cf66)e3{?)x(aVW7_*1PcQbmZQ(A#6k$Dn6Nu>$?|<;I-}SuKjiP`^<0r0aekMQ z&=LN^ZQx_=1s|=&auDg8mCujtV-`Zvo&05k%U!3WSXhyksx*Z6J+z0DTuP%|%_B*q zLWfX2+Hac!2j@7RR(-k1B3h|bH@i5mi>xz4Mm5j>1#}7~3S7W4r!NABqrvqRul!TX z->o-4F<3w76eP|c!H=o}E^v6487l*uj{b1!ONCakR=uTLoZL)N#9ChleP1fwb>M@R zy@rH_kWjux7Dc(t>;04~4Oky^C{>;i4N_jN!O%|VpqQenC}ue(EX4IA5054|!pYbJ zPa-HVu7}$&j+^uFgIxkL%GBoOPcDZ;>&^Lt>}RIN(wW+Z>GC#JzvXetIa8K>4Rq9a zvE0eYNv3_!%6+dc6p(6iUb+iY2?1BNc(pux8ym`U8GT0KPpb-4t+7#YN8lxmNYJ$m z1nWi&j#Edv#ca_20>kRnHB|y=6~8iz!m2^ZB24UEKO6lf*lx1&;_#xABa1tz%oQq6 zew#+=3PRZ2IjC7wDt+O$3p@at&kYvF=QZExojQL1;6;%53i;Lr>f}@@D z;B)bbfe=b4QA%2M@T{>BBY6dtBBdfyv{1h%c-+PMD2MA9ozE??-SdFPleuBfOmgn}*g|peMVhp^BH*sQc&bn0^HGg_FcyNy7pp@s+H{%Z>xhMZ<-1 zD;q|3-2~7k58@75SziP)Y4ekG>CmJWgiDxW#B{0J*BQ73JL$E!U9Ukpbz6<6*_wu7zDbh&*^Nj1sn)v7ll!g zm83zH@XcQ>=%wu1HpKooJjUfReF76YW1OIU#ocddn{ij=F%Y{LF zOE@QXWtFd|v|rX#Zoi6&1FRJnihMW_&ID@!F(=n~a zO)XA1YirnENQ#mON&?Gc@C#cbO&Zu8hN45s{mm8|o%~sYgiYsB)xpGaA2=fpayRJf z{@%&X=3zLDia)3_l2ip$M}#woPgT*+EF;@um`-v1I|;%qI)xzpLHtX@e4>OjK_UaR zND&1^B`tLVZBrT}U$7VzNMclFLI`%#!)d4%SBDl1$DR2J`6kD+F59%tpG8h=+NT3W6QxN!dH8ww@m^R+wa3Ph>Q2d9%p8InL zyGp@ot$Keb!s`BF9FIYO2_2&6%|iVE<$(tL43Bw)8fbCoPCu${bw7hj()XjnkX>Fj(7MnmNb#`U*cn~4OXAj<|4M2GZVlaLVg zzFD~-{|ccTe#*;s42ZGI(YLfcObb0jjMVX7YKn>h&mg{_>fld0yK@V%=m>aU)MRCS z`YY5GsAiI?(fL#!Oe7>|{&WY2L^5e@WQayuhi`8YyAG&e+FnjSl%_c8O)Bq2 zv!z|+qE(BEHg!rIQFH|FCDZlqhGz8pfSO6mfOa&i?SVS5W0dmMxS;1SK zp?scFc}28~iyJXmK#w1HW#lL5KEb~D`1sCVejSzhopW(z113%dZ+wWn^=jXdZW^ZB zV_C=Z4ajxzb}}$%u(GnzG_Jbd=GIFp+Eao$fMC13OPA~4_Ak1~dsK$Iaw`-lAkvAj z&@;d;D;SVq6Xu$zweK!&+SR;DmC6+;A2b?Z4+8!sZCGUdc!`0HG9Pv8_8PE;LcwK2 zO_ifj-2I|W&WkW@)duU1pU`=pdyyW5geSd{7y&N<+O7`^1?kR(nGKgW2GoYbsnfM) z!YTwdHX&>7FJ?lIHb_ZPRW1c$g;>1i-V$_S;h`@VPYzV2 z?W?QcQ3d7`ohU2z)Z$@<)nr}`n@7Lr{Xd_1aoXLISm9eWSnF;5^nARnue!9guPBgA zS>HQaX;SV_Zz0#$`^jdpI8p9JPm?t;I;htAJhfQ`-5BtB?yC_U3XY1~*1=AEayq)r z!(pU_o90StrF_RSxgl}n1cIL>Z?!J0xdXI0?WmV546V8#C=QV!T!l(S-@Gg}xYOWy zC8upes7S9CuavJJXkR3Da$>ZcQkjeE>yR|>i}TT}k4jbB!5tr+c4&vhA6^ePc&twP za&*@RS8i_Zc%|KRPTbdyPlZ7c5FC`PL<%Jq71qJ} zRd>E;f$1&j;ls&1KS8o$n(}t0#FAwYGwQI}Qoo7_>=JVQ$QjHj?0@of32)fM6qj-G z_u$Ash1hP*osJkK>(=r$$j9-zXF^;E!Fm7utBTR(gAw00+OM)lLsq;8-ep|Afy)$d*|;0+}OpVnq2;PT&-{A zIeIY-4Y?9|lWJ-Mfp90xHj~Dlktm_8cDviT_uD*fks1w%Nr)My`c^P-(9qC;lGQD) zpJpm<7IJ&SPVf1xsrpouBHxHghV-t+6$^bcMR?PAX&BYGodnegH62i zspZ%Z)9pXAl9&s&4R>B%pGKEceWR2-_05bl3l-uMl=RY9zkPe6(X5-V4&bJ5mKS-7 z$${{=8cC*#FdjLSm2Of)yq-Raa-UP8KB<7IJnb`|JTWCk}bU0m?jTB|j!NJ~1vHZ4!Fe?x0yuq1o@w5)vZ`jrF7qO z4PYW2ED(2g<>Cpgu*iu+mVJr4qoo$jCMC!|;CjKOTMMRYCPqc5S-AMBdaE@waaJxT zs0)O1pOdU2g-XkscAc{y?8jNJWAy2KKjsNk;AP56nr)YM(-syUfIac7=)FFJ3rRWR z&9=>RFSxq6=)`*W;iUxDqPYtNBbcN$*a{bE4WFzu zaX&)^A0Hi^9Gxi!h!}h}=po&gL%?G@?@7oQ7#iqdsbjMMx%ha*LY?MOorQ13MuH$s zkCefzQhXX$y>`WV45#$4`BXkBJ=o96Mod)8jh!-uqDtfexvYJ}NyiBon2K|eOhEnP%c1-?McwL}`PP3k9R?`i2Hf*!;V&JCu~Ek-fNZzK zEUf3XXUAZt!f?E+*YU@5V6%wt9%p>~uM0XDlA7Au@7nlKE?xqzO1H<9FhtqNTs2n@ z7F*KJTqLuV?Q#NO=G4c#yP4r3pgFFr zt@cTPV&1&xs0c=cuMf(}@vDuPDRn)kP_Wc$Rm%%(>zNopDvNtmcmlgazx$$-HJA9*W7LthTG3p!m<)v3YsMY%id=PZq5X7ljM^K%-xocsU*$zkMMX|k7AgZrZ!^ET zN`0;BxY^zHpH?+7AcG92*!O?;t??Q#2Di{LpnQQet=bFZPBU!eXG10TAJ^eFI&(UZ z8*qt7Nenu>-+M@$jE|~yujs8y+pjvvg(Xg#eULmf1Z-FR+o8OsWubgzLe5_7vge`E zcy)2%Tnz5}MFNUB4(+;0b5a+P?ydc&=^S3RAvcg%@lfh~7(uFvgG`rTzB?bY>-UMJLGF+TOjy8oW2+uS2u?#b+_X{=aqCtYM?`??iYqOu&`l=ac0}WvU*Snu(x&2Fjf}Q_+4ubo(>j<|pS8$g+|3zW(u|u&-F1jBWc5~E z3u)^d)%@4QD%HQP3o5SR=JxnRgA@C4zKvtQO0i~>ll+TUxB={op|0b>LtB9H3pYH;>cRpYa03V~_>3c*ksWrCN zr;GjZRp9S1z+OZ)- zu>X1beGH)c1)9&uWgqc3Gy9+Wd7H~rcZV}9P-To&zEMfq$e;a21i82@o1r+5+ z4jrSJ{uRT&_pt~8i1qG|v;H@&}^02s+si6sQ(xBq?i z!TkE~#KPe<|L(Z|gO=*MELB(F*ixM$11`f3n&`pi(dTvIBIVp0m}J479kg+)5Jo4MEdI5 z5fdgV+=)!zXRw08pQi45ZFQHir$S zmL?TWN|e1@r7iwL;C7YSCWxLS-w>T0#c|crk=X51)Zt2zrnechY{>LqAMxpZ7Yzsg zpTC~R407JEf9;xI(*bmpm&Bpvm>#TJy{Bg{XIY&$pw1KvUJGNqF>fA6((~6|fB^Q8 zRSx_v)^%P&@6QobwP-^cf<%OMd#3R6z0hJ*UQb7t`QMNN@)UOkTm;M4zfZa9-?TMS zNVtAfKJi!pM)<|pDRodBEV%tel)s$)Hw)Lj7R?XjV>0(K7^twFC%#cD4h-Ow!*psp zZ0PWVrNf277SJ!amc+EAe^ui{1^_Wg#BP#qygQ6f(^sHb7zuOJy!`0KN(ktT70yS^ zX7mrzGl$gdl6B9DwHm8s2n#!yT875(esBm84RCOTb;$yvzxRj|P*(5XoWW~^53uBo z8&VFfO>NvX`aK2Qvx%RGP`*cwc0U0Ce-JxT@|+(Um;Ab(Qfkz;?4D0a!mgg=*96|o z2>AkBj_RR29HB%h72bY^@|rJ>?C?!*|pue^6)OUC4}O z8kP_jegw`2D}XuJt*xq2VZ7@!^8oU>>S=SZwl77hRaFFUsr+W?Do{;*a9BWH?&mvI z>yO@ZYh#=O;s=+SMCYA*Znx=?P2x$cXXYBXrESi-5(!G_BkGcqAADGA146}a>z_0Q zjQ2YL&d|=Rh30v`x6@EUHqvwKPoFw%cM;BOSittz(HP_bCX#p%L0tPWaigZE=Ne99F|H4r8cqjD*U$^55a_Qt@DmUbK85To4N)mN5z;e{ zWV@-C<7>OA5GEuRzUH#MPqba*9XUtm6ng!Eq*zv z%)2n5q)YbrGx?n^a5V`4%BQ-@CYSsnh&f0cj)e1N@}^TY$KCXpg6fe!^#!%om8^HZ z(XStj7`|JWp_`e=2NI^eg}ma-{P`;5wDrz4F%JB^y~N8+jUPt1dNTRfM^Xep(8Vkm zi~WPnYhNL?<|2wHNm$~KY+1x}b2D?b9I(jEBl%#4=)4}#LAa_9rU3bH#;+uZj{ibabZEyJY|&twW+FQ1{V3(AHNxTVC#CZ1a}5t!I`2m%67) zrqk0fCNefj_a1X$>f3Haz+B(i-@l|C)X=ML{o1Gc5WVlr|7AJ@$esRNb2-hn zvyaKe`bpGpQP*`|vmePasii>Q38%?*4mwz%_SRvw!JUi;f0u4qd(aoa6NLY}v-U^i z>wQ?)E0PbwT-v>h(L7(Mt5J;5jV0$*R&@GF^QpwJMSibo(pg=`)75?LlEfQ*Y{co* zWp$7Ks{v8yKad+Z6hQ3Pr9=IQx6VLN{(=UXUcfEAZZ{KA+6 zKUt)8pa+jDn`6E^`;J{GjTH0n#-OpRK3**OSshB{aEMqZYlB%H~bz*f7$6rtb4T^-j%v zroU4su->#CjH$>4r(6CigUy#9c{XBSRox7VIYMm`Ri{M}(-a$W$JXh z_+ePX?}l$^nL#GvvG(#Z%4&O*pg|vkL&K}Q1N)U7Fi zujW_5D_UlgkM+p7>^^M+5BpieSxyswo5tqhSjldWU%H2EVt!Sg_q6pioNA z>VAI^K2;W~@u3NSkd%Vox4@R#w3tR}D@tWNhH5?}AKL@#HGo!kTibdNG=R-uL4mEX z+IbtygnXm9UkPn^nY7+?-R+|Cr;Z%BRy3Q?cO#1nZ{BRHDOc#Lg&ncxd)f*eEXgh* z>@T}+2gwx3$XMUT0|Zf^Nn}6d^px&Rskw2-LqnCRiuhX}nc>Ns8QHl@h{TjCr=_|1 zS*%;JQR8%TbN^%}`yzZAfPE9NRaI>)Ep^bzY*U~r&8bt(eBBdcx_31cUtQ~Y-s3;D z7#=_Wfl}rPb!q#!hT?Z-5prsK>ddt_KwUWBHKOp~S@qgoaq5%*Q%N0O`z#B28^pzn ze)IuFA>z25v_&f_k)xxp_o!^K^Q`R(OM|_C*~{R0kF+xyg($3N(<|Ly580y*a(e|)9uzaVHCIR=>la8?Ua=(MX7d|2rhpyohQg%ZRGbv7Y;6N za+n%I5gr1)rWY&coI>8)(G{ojd01g?QdsIF^h~$%tig6%ve?F%b{fZTsAd(lQl)~a zD?g4}MFU3|qssmt$4KmAc~zO2GUZcE4P5M6PBUn>8EH9W^pupebX0**d)vpx1_lO2 zAW8~KXxr$X-SYrJ7&AzS@PQ>!p!%og(?zO&pNb zXrW0Sy#T4vH#p2wxy}r2kvCV3D%9ItWr<8-G29=R)GVHd#Z*>alz_14P-ub)hXr3% zX4d&z=$lyqirXR)LUxDK4_3-AC2~c3^v>qnzq6QFYdy>!yKZ8uWK$AOXkyv3|KWk&7OGK9mIMv!7 zmH8J8-6G^iH?A+tc?kdmNJ|CeX42B9jiyVSvBsVpG>rAXK~h*#1hM=x(+i6fP9N>o z%FWn1K}TFC6Zy^Be=x-W35Co2R-)v#zYYSHSIqn3aX7q@W5h5vH9ArD=DmuL4-CqY zbg+Pg1a<3JM@srR{|{2MypjeIenwJSTtZAt-x?MQGzOBBk7J1~{2;Uo}b#$gcMGp!Y&2*a)5lB``Curwtz#Qm3N_cyIs zi(W+Nam79{gy-UmLX28deiC!)ZhgixGnhc?)$0Qk0Ong}5rDB__4f5WMf157yIGy8 z%pbJqnIe`T9hBIiNu1Xo+W6h9H)Zcwj~k%|SOQIC_Ze*U5f5s?@wKJtrJW+wsAI&g zx%~Klcmt!L9!DHk*p@3EW!|2EC@2n&rY#QaAY?y%!jZw$|1A#29To8{$*`7W?U_GW zg$P-cQGWi5shjZte?{*vwD!bXbkRvbg$EWRHjCuzT|9qnl5Y%Aixvg4{lVnv1t+u2 zrySW$`i}Va3S_LaN7q?K=0kTu_xM?z`={Dc<4c`(exKwoMYTl_{cY9!v%KeCy$v%l z(MY4F+&8@aDqipkM*5qlP@D~fFe_?Q~7B}htK*w0($$j)aaf)^}x}5erRl7hU>&af{tzq@mBhSmnpl_Uu2L~xR!L9lYMZHbnc;F zm<@b$s}z+VN1vtJ&M`!qF5LPT&L_&FLad9mgEf*-6 zc0e=IDQG1#8PrGOQ>z0yWQ7hYVYJ_?eA@PSG$}H4R>4nd^i7?43}}A!*WRkik>q1j z+swN^V6$g!a_sw#d~FQXYbRta<&zNyOQs{~=fVER-1Lf$Upd*>kI*;W_Oio~yRE6# zAxshVu8_^qd=2oP&+2cbVfn;M9^3KdUMv*-INlC}df%Ln$UCg+N3H6lSPf`8+eVNs z2nXl)%HCoU%)`5mz?mQuboxsF)G~!d%B}yVMFHSQXoN<5j1l{X1TlE4Y;{qR4!PRM zw7#A7D$qbVomUcokwgBD)lZ8G)|W6w?Sa^=jl6WY76j9${J z$g?#i-O6Kg{cBo)%VM|t(-l4fJhY%y`W#Z)Wp3_tP8Rv=^A*1y4c;7q{f#X?yWu%; zl!TL=1FMj}YS$CUk|+%&9o@^umQ|hQYFtX%*5Tn7%gugKgs6k*LO(&Po z3wU^^X0ztYtuOR>Vml{0yl%JKP$)u;jg99g__rjFN9DxWUTai{`>SqmUUG%oEA@O_ zUWA=vV^=5j+O<_x?iLG8$RVAhy_ePW@m*5uOa1-7i=)O|W@i>!z7@m|t*MeF3{*5u zT<+ip1*XGZU%9wF96jCPuhblj{o1afa+vYIAkNU@s8=l3*>VrT^*Z)^YZ*H}#w{4b zd7F;4%M$(iwZr!l{ep_B)4{B6XE$S8`Pv`Pm%40Uy(EQZoeg~$v|$_j<_iVE<>krA zDq8hdjhCxl|9Qa}h4rJoyEb-h^QHWmV)3=e&~R*4LkhH&KTVBTXI2&*_M<2bs0nfL zn&!w@Q4y!dr#~!^W$%2VozP&rB7cxk_Q!X{kg@8hj3p`h+}Bor{d}!A@pUhasU*In zjIB~6r#bugZ?G{_TR>C2vQmhH55!jum0L!&f2Vg+mjE;cT8xA!97kuky|=g5dB7QD zg34Srn?19)tCJtr;>X_3P_CS^k+45M2fP9e27;KFC_Oc0>|ieMYI;Aq%f`>j>~Ey< zc?y;NTxl&WrLw$}dNh6m=wXd6oyVbSBY`q9pajIJMuxlFySqcQKT6dh)TdKW`w%=( zQBifd-QvVPwaLq5maEWA>?dLpH@jq`%g^a}GYt$3Jh*()OG51;=WW%RRWkqmDM*%4 z(Cy^hpV2XYfQRLT*n(oLFf`W@MZ|$2v(K;8l!yZVu=A z^NN}uOKREcm`MvEPy_QWYh!CmGd-Av&SAgM!_A`^n91qs*~^QX1*l?OvSaL=x-=Q& zc6Ks3l2kbX=s66^E{c={r!chiQ>Mahm2A_cfLfUn?|w*#imgbUdM?PC>e@ouSY>am zkakUCYdfdIG6Qi;B%gdj?|67ER2mi99?!wXPZ~qM@yL=vK$4lYad|B*xVC{wHv8it zwAgRgmzSZzp_0KN)U=eJ!}1QPUds8*|JXBLt~hBOP1Qq~8cpx^2}tv`?bn81n?tB` zL%6xIAk5KehQUYip3hCZwO0OEZ|8LZ!&(B%XxcD0@AYwSuJzs~wl;K%AO402}rHX}1*VWrhQWR&GX*^kNyY+pO zZo6GWU#znTA>k^e5GeR8^Bqd!wF)&)XL6>~#gJe&+v9+~das4Z# z$c02J%(aIj(%S`*V0!jd^)Y%M&MHG_PN*)QX&u#pq zmRpn)j51R8j*tBl_jS+aTitG?v1v+o>Kv8MTHXwCK?BPp#X`kbJ)c^B>(5tTn;qFgd)(W!A8FBk+ zU%%SvY`NZPEXMa)FGOQGOLKhs0EJ8vOI>|tc=(v_b-u}M?{)vTbFI|TzRp~mRXwSA zMi&@&yf|ydA#4Zvrr0(9(%tcb0|_-ZLYZkmK#68k`$iXpR1#Z9S64(IWYNyfX9#$l zgy(en_S9qr<#u%AiHKu5 zGiLlDlF^5kv6%G@TOoHGHlXMsU8j!FjX*+9Zt$*HNxK=tM87=orT;pJJaP+9r?*wW z9?%Mq1}wi#W2v_0$RO@0*;CafQVe>LX$>-JWqK*67_b-!UBjg2u$-4!udxMEKNes< zq1nP_t13~<=XQ*ub9}_)L-6!Pl-bW18-Gz%|>o4imK&w z)DrU~^W;JxRur(IFb;nCWJ~6W;kt0TE_XA23%VN{98|40l$Q`tQ&O7PjaYiRJsIHR z2Z~PZ-*FRruce0Nzn1V8&}N-VON`6N^_U<1yy*9-Hv+k?0)>z7#8oJigios_WoKJr6qy2%-XKdv*n#N`R{)L7}X+wq0^H! zRI*gL|6HpEp-k5~?;ZAajS@`!HpwY3JDZA>6(&n@N9>vR0{d*O?aZf5po>mWNU*!1 z>Cdp&q0KW!6h7Ze&mLcQpO%x=#{J-=(YTY9jPyOrXhsXp=d|CS2LUhrxdIX_b#8KJ zc>q=(iI0@@Bo?7e9CeIH8ePK zY;d@?;`goQ`DZer!qE&UY006W!kiv?%LXdaTeasx`3>cAQXxUIT_eM$22S;UO?i3d zvGl0O$k49){;t@*PAhM@ZFE#Rue+@@Zg}s&mKMp1sEhy!M`LYcA*FOgvy+Y>7yP29 zo!_Zrsl0C+q4c8}QQgR`ye-`NYF4IViwE-P#AO;?-PJBIc-6HAmJ~Ip5D0i4XDefJ z@>MdjPUdTa6AN_Qw5k3M}>;^sRIb)|9!RDQ5i zXh6&T6xwhw_W&vrSSOA$C9o$jLQntaR=M6EbY_He{L*+qBG~hm%H2dbx=N)%27BWS z1FPoky*)oJSD80J)4`RPxqyO+ijA7PZ`G&K3>@zC?KQ}%p|R1ycnw^UjS2c-`D^*q z>FQ@J^aL#Y3^35Z9bY`)+gl_^8rtn@yrjnrmnc0>I3lN`)rF3Z-Fof%?jEurJf4LX zb#fL$iSMHiAaN5?L}sv;7Hf4+a?bWcN?++)o~OrkK^ZMTOZBaXyaDsuKHu0{sz37~ zt6(WQf!J0!^5z3Gj26nfyt?&#@uCOBdPUtr870akC*$hP7W$v^Oz`+zc$hAw>h*{< z)%|#(MT*JSx!agg63K7d#CWHc&h9DIVRl+tykfpSDxUo#iJ&lDQbNaZa6O9nI2H|s zk8u01gT-Sm&JCMPTwpNh%|vnP`2|M$b$%J;=tR{}#KRK$N3P)}NTK<%bTN^*Z{w57 z8TGZ5c!0N=5ZLdccwg(}2)Yy)U(?fh9ZsWY-3*l2czMgpTW^oOfjw-{W&8=hJ<_>c zj4&5;_Q+wP0t=U^u>P5N8UY&>%v-^+e#+PlKe2rZIh*=*0-^!oc+2fbxidQCL_#y+ z57^?17>Vl$9(k?U{&+)*3q27CT;nkO74rR0?NMvbFyVgqg1lT3P8`M|k$bn<99`Nv z%j+68K1pvu`J`)`M&sR^oS7x*6e!_UA_!uN*uY1dpiE+E7y3@m@HY32h z=vhF4R;`cEeMbk4*6GqoOs5;7Icm;sxw*OJ+Vu{7w{6$vvQV#zOW*6g$>P*0WMZ;v zU0+J`G@Bg7L?oUMo*XtdCS+rvDkq>&$f7XWby>(BJ9}!pw2h@KCjuSR-hkeOln_- znR{6ShO@Qbczu99A^mMzT%D|@=s0M%P|2*jve!W;d?RrffAJ|7u>#JOdK#sF6lmyY70+f{wz@-d7&RjhoNR5o14kil1IS48$h%2(5+lrRIJ*MEeDXmE zF)>x$9Pv6)YgRq}02W>pT>CsRI8xpAa17?_B|!7AGfA_}A)oXT z86ffbbV(NV_hADn-SJ~_%7nC@PMg-4m@lPp__{C|XyCXg=5;caGw8}`-BuSCklgPn ziN1ddXaEyBF8R=P=+KxbU2%rxLaV;b631@AvR#N2U;WTyH1Wc#qj32u}L;HX6#ZP!-bya00 z=Cs3#TeohrNFqmxCrueUa3HJ~ZgkLK2o4T-e({31n3wYl%8h!`$*l;4UQD{xrGFJZ zX;3?Izcfhf`RRR4V!#|cpYZOJ!yC-vwgS1ns6d#RnwgQ3j^7^l5ExcdVPObXTvW@% z`aLabo$9$m2*fZ!-+sO3&71Z4XFq3U<-fQ5v*6&CXHQ+0$)zL544FK2JZ5s3^d~M zH4AH-DbvO-S@JX{wm|_bQ*p4jEh;DqJ#m4xJe>MmK^l0$MLoP_0>hG*pPo8(JhW=l zCP*w4z4hitm^Ky=?ESyDmwI}*A-q-VKx~<^W$V6_sfq^sB^4uTX z?6U0K{7*mr=Fp+@R+jSVGsiAkwBS~rw+4ztK}4X*2#^8OWN1n7-6g>K#GF}z#tmbL zBSsEf_{s z#bI3_9%8b_tfoCHEGW#*%9Y41&Ylcgvgp6K24Q#{0md2=lhQR>eSThXN@_an8BR}D z*xCo|A++>f-fo?{h49u3uA*F+73SmT1#e7gNmXe{37>?IjAnAmw23Y*PRv~42;eHi zi2ek7^e1CtJUC4`jaj44f|(iy2FJ?MqHWt&#+aD<0bdFvVX+W47DF@~%;&`LGJBGb z$;!?Bx57GCUvJ34fiNhHClBcG%pOLRg`}mXV@y<`kPaBo%Q)f-g>h^A{JgNJDL222 z#pY}z_D)Ab1kf#RDEME8Vfp^QPV-_8|?>EU#>c1Xw901}|EEjW z5UyHa)sBOM{o`|ItzEq-B_lr}F%^qsSTC8o%uFus7jDdp&zaRGH41PiEL%)}YTK?g zd=yBEz`#I18#|lQ>>}7C%nd9MT)7mLay=VbhO}=zVNBybJmH9kHqoShv~ zlQX^ie6Ym=izX0wdbqc1-+K4nqg7>9*J2W3!0`2{i>os)F|VK~C$Atszc3~tRfT7u ziG&43h5RKaCkG$z7Lm~jQIRo=o_e`QuWo~f4d~Xb6KoYeLd`2{8LnsMu3Y{lRv}{B zhWQI0h35^oz9s^_6M=h4fDD*>NuTmIJpr^(%n>5Q0vB}6(Ow#}K+9q5rn5i;_9kWx zpwXMbwKr&V@e~)B9hlwAHZMjivg6NvOJ97wIKNb-!Sp8`F6ea{p;#fdw9x9cIz#oG zD71*!Jpc_Yaxuze8DTaB7M0j2kYQz26^0C1APy+R_FU#a2(VCxd4+_cxcG$Vh)8ZW z8af$hbuo`ZERso8QcY=DDHppY4fla>CWSCZ3vba|^lFd+8!Kg<6J}fBz??D>+nO5Qnj4d$kt5n* zyr49YT_c|>DtC8y9oQ@602g+NuGOlu_*laPG_a|#^#UHl2Wdbf0E6oN19%CXZ7IUU z7wE(}caVlt>qZ82R!!h)axS{XcaEcbTQ4kFzjHbFtAvLKw7GKw25=6CC7H^A;atsZ zcw_pblO~L9z_Ej$Dt9}$Un3&kd~fhf`ZwNsB_jOVh09Sg8>>KXk2l|W6)sl%)*Ae| z@jpTC_X~qT;O^n>?85dx#KCF?IF_+hh_Xr~m2~Oa5m_PJm?8FYws*1u~? zuA(}CrG-dbZ{8Ra?r~H;xPa?8s>8XMSmr0UurP|n30EmI8HkUVl@%2jMUqM+E!(%j z-tJI|mw@XqvAT6`w{-*jGi--J^i+*Ot{R@`C0-fKVo^!x5V_FPX+!$6xrC&cn6p+t(ZYt?M5 z4O+Q-xIO;Fte9(YDJj`UkA)u#z4-mtzqot4JhS-8Y11e2JI=9h12Po}b8`wXn_46m zd$(|Y{)MM(t*uZrUWe*D>5T~7BLZZ=+#~Rmt*Hs1mPb>;df9BgH!61aWD*>=$_M5I z8X!y_u&o}0M03R|Z5Sk20K=z0vCNls3C@`***1;E4I<=N-wVb2-1n%&ak-FH6z zkte~ zDvcTgwy-ZS*Q+KX65}$nRHSklCa?41>=7+!aB)+lHFtyoW{xgclFT})Ihg6ga?rsj zV)h(%6t%WOT(w%2n^V1m8RD{7A`vD_u#OHa9n6ET>2TCYqW2pFurB<@hdVf%^$_cH z(s3&SWBRpzXZ6urUH#|b$1Wu=p3&!@$8RJi>xv6Ra46?vd7D6h$(gK5sX6ew#3^3! zQ^Py^?Q$P#uCqF^sJNs`U5Q@1K&w+#sS5H6FvW?GV{HI@Ioq zLlt;a#WIPm3cuXC7BzBa%iDAsjJfc&96Zpd1~62@smF=s6jovnWefzNU^P#FVw$rW z6(J%y{3o>zY#<`QhKl&d)ia2Ix(MZBCB|z_y#Wb0*w%|&wZqlQEP~w9(x5MqD`dVw zUJmwbu|8j0q1HsT0~7&4{#Mp(UKuu0flA!SShe?trRTC>+A*g->bo0bYW!9Hm8rpN z00C&gxKIS}>Y<>>6dOq1Tveh;!GcD;fZOyLA5NS&^X7k7T#ZV=jNC5WLac3UK@YP8 zBBSH6lBcGitlhzcu|M~2_+|C_Gbhg_CuQd37bK?SELr?kbkwz1UwgsY+7b$&B}4^n zVIlSP_RY@9Pfp8wcF8&nu?9B^XRd|v0){@ix*^~bB(r&%dWaX z;#(T53(3yT!zxKXKQ=82F|isRTh=7TC!tk#b!F4l`1SF8OU7%5XjniW5a<&dlPEA~ zTev&pDJZ-^sDt^+SZRX|aL~HfDU0zVl)yHfNr`C}E?mZT3J~I zECB}x2NuZ96Jd5&-N9B~7@LX)y8(}jL2{$~vf>hqdg2M5#;|HYY~mKrp*k*OhkaVn z+~R3g zqKk9xTzGD7J~}Zh6(<>&=%`QNXn1n{z}pBJ(k<{dQaw0HusUd1kDx<$JX;SQGt)tX zO~QUse21M42^f2_#ys%bK?G*P!2isCsm3csg~gwJyec*>#X>5>_(NJ|)`uT{X>DhP z_2n=!>WkuKBRz9sFpJ`bG(<)waw#TxNmb>lq~z3o1A6lr@i}7j1&@cqo*bwo`I?s1 z!y@)$$A82$Dppon#yal!=N^4^Sa5g?q4X-3JnR@;>s&3 zu~-pe`GR2JDJdz9MXptyc(m|Hic9zP^?!fmYna8#$K++ho`6gyofGX7;2$5Co|}`O zmzy6L=+BGAD+~c_k)nj(iIoS@Z+&p8qT;M|2ao{bg)nQ>M(=$s0qhd@vAC$@&p-BF zjZIcqD_(!+g*kI)BN(|yMO}Mw@tZ}LV8B#kL3}8I1vA(>1@{O0yPOC;wQc*pyn>QH zx9^)caZJx%-C*bO~f4 zx^B*!$zgxhZ4%m4H0+rEYhzpe*giBE*q9Q#%5c4T7Lz}?M69v7a3SKiwVTm~;mIU8 zE?l_u%TMbwGjeRKEih{W-EP(%Gb@SpBv@u_*iVb4IKzeyfcoP&|9i*V|*r z&;fkbEK5%B`qHrLG5k)LbXHnie&YCPcyAB~&rI05=}(N?p=ZDweoz5~VB}y99C{J8 z#|4>oER^kYbd!9TFz9p`C-renKGOnSi+cBfYvrp;I%(o)Pj_dXR)6ex7`6#YOiWg* zRHdcmd-fb!{nNTi6~=4XCILutQ+5qedN9L^N~=sVuW5LXAPdn-QOCo8Gnt7$^Yt*76{(Z;PYVDB0y_dfB zw2hS_A~NpluYO8P%Ro}ZX|QO>g=NGlhwAYiJ{o?iojbR06&xV3P<;LM&tYMgc_EO4 z?ddnH-+nzU2O~RPJ|3>_YzdH+m6c4!=9p$>=ffF}5ZD+PQQEd`SAIzaYEV>YsDJo_ z>4LqZ=j&n6>LMazu~$1b;6i-J0ig?FS9WaQBf-NoncCGtMvfeW z3N0%$`_zfEXbz0RF&FN_xeHVueZ@hN})@fUpNOb_lzOH1FkKeVi( z3Q%s=tXlIMZ%~z6Sja2P-!ujjF<9-%Y{DA{6V91%To4u<78VslnT}oBj~zD(VbBUy zbOu(j4SM<2MdKz6hpCufSd7(2Oe_v)Yrxb1 z3x!?RwTY4b^_ly9wM)wRoWo%jkd|i)98Yh^i5E^<8 zyMD1gGzRjqP(y&-{e=ok*)z}1>(#p(>Sp*u*lZ{?3O858XkuYn;FTAM#gypJSE|i5>P#DTU0QOscJ&U7I$oQm-FZi7oM9yL0Em(SF5UjX!bxoP~`wo~2`4Mvr8y z3~a6kTE^N#AqWRR?gBU83{4#?C!#V&)iIg%hPm0GcT?|zv4BKupd;cQw}-S?0letN zC;zwXgW}@yO&fP%NwJTQS4CL`?6lHKJZf61V}pgvexsv70_U!#HyHkJ14s4>{g959oK>YS<>K4B5do1+S>Yq1eXXkr+xhz^d zAFIZ(eel+;drlrZw_)9u=bm4{Z(!AcOy*x^bFuLI#Zfy#V32>`e%)hYldoUTTlU)f zFD#zlp<^5DuXym_@wIC<>jV;kUNw5`P}vT~kjHx5$l5qcV$7IDW`8Ib*2E7VF_FIE2X^Y0k3>C&Yms*&^O zFR%J)by0B{o-D;_jWp~SKYrw|KW|7#zP@9}eypoRT0EXot5F|6erENL8(?3p_~4CB zojV}r;K6-;d_B`M^Y9E!EJnmO%6P(kes2Eyb=yy#ya4x*5KB^O*OZ)m{)MN+*X_oN zxz=?y&cjvIH^Qpz8kCqLj7fb~3QLJZ3Jc_VdWMs;BU(SWx%U2bC@+tDrZZO{3~XUL z%2nU|jE8iM8$Z&sg{Q(wfmwpsRT&O*TL)`bcUR`ZVihWLnZpH!jJ&;Dy#LV~PyOem z_~gv(+yC+p^qnzt5}qK0s``dPbVda3E&(!N?k;c2t;q=RRX-MLs5MGeWo1fnadKiB zYuQ*;%-V1?rZ%?nNt4H73=baq;^Jbp3J-#>P*(9PAgfzJjSdS&l!?z0;Q&zO1pe_nfM=k}PdzWfn2D|!JqHOSM`t4Ghy zqehRwG5_cZsN~As(HR0CoUl1)NdI4c`sIUnKEo;utF;(?xjLbN*H66a+6h;oCkCA` z--8dXVMC|QXu}|4Sw5%;v7ln=U6Xh3*?Z{FvE92e_U(%nWQoDJ;gPEV+kOU8n`~3-bXYHsZ?O7*v}^?83n&mV!K}e28T={Vlmk3lXnfQ5obZ!~3ZU3<eyLKP^ zZ0KSB<}X*RFlPuiUV8^Pi7B`a*a_7!1hKLkVf= zSs3xSWy%DctB&yx7zCU|j89sD(qQpKyLs!DT}UqWV62ZnEU|Zbej(Cuvd8cUPEe8d zcG(0C9%c`2z~kCEyZ?miuimn8>(ZrbckDdy%+s%bpPu@T;eht-ICJW984EmJnTaJ^#{=bJQ2gk?H#=Wc;aH+1=$hg z@V7^$x>(C>Q@x0wkf2VT+yDIY%DAJ)AD{i=$Pt6`3JX`Q_+#7FoiO{D4dZG&ZYC_k z_WjN4xBt200A~AJv})0?VK_$R)~woql>j4#UEH!|Gi<*^P9U}zh{D$@k%rCIzJ1#} zr{6Mb=D)Gh?BfsTw`$qEci$eUA|cmbRnboxfeVDd=`7j>5_^QCek4(cr z6t|pn8eFO@-1uNLc^I3uXo70YY%pFm{?g7JTf*q$BraCja$)%D>TzQ_b!>-9OMUVT zyZmR(n$f*$m({D*$Hpbrs~0k|al^~Uj>7K7Zk<{L1O>Kl-&#elS1J!6D#z`%a^Qi`*(4+;vrZsNF$FYep0VMDl89e@*r@%gs4?b<9~ z_WS<*`>}r{KQF&gWJFYxC`{ep3kJ^U6#?T?IIN2+F6z~*%aEb{)~{KgbRrFvcAb#G zu06W+AJA8Q6#?}i7KvRvpf^qiz?fdER?S#7I@m!G)R#C*^uv$eTDagF41A`hWkg1X z_v_bd;Gq89yR}J4N^ae@mC_gOp=-up-m6>3kUBwOp>-TQxW;|rjpGLoIT2hZ7}#<` z%V_A(sqOtUZv&Y4tS2bJ0bBE@-+L=s8};hd@%G`7I|>9Wb-QQAEf9kTiBdRT5dPNN zC*gx}f2>-Yi&+;7j+^%G-n)BfosfAS&PNT8y4~64a7v;b3V~g&=)m;v+{DkP+$pOX z;K~u`-9EDF9hkj`tGa{?!l4{t+@LT!(^it7be#QIHi#>`Z1@UpUN&ur9IR1TSop;3 zC)TcBi`At6`qzXJBL<@wqM_Wad*_+6?t1j$XAkU+o&DI;n2VS+>00nasN$;JxEEkt zBQLWcFX!Y&I4SrTkZ9Aom0*Y)vAgiD#7plR?KsJ1Y@%1;K$57FN1>YTtJrWyp2;w(u)%2McW+WX;dSJ$5 z<`O@gC7@)0)>mCUrfz8Hd+&a_deyr4xOfb@SUk*up&?_g7(8Xl4OkegM1^UcFkvia z_UF8bj^E+q@rhW&6&Mse_S!N1F6#NrlP{+vWMODVNn7p?4w&&MwiXoTXC)^c!yuC) z5PzIU4drrjGBebIK)gfrD6;c#XiI8wzAY<_=Obe_d)&2`!|UxiAFf!o;*aI4P$x8t zZi*Uh<%%_ju>eork&lnhgzLunc>8|z@%*h@wr^a!(O|ZjN)@2++on!(p44o` zx6tx2$b>U#h#RJEMvWeh#beJt{VF2Py#LWpK6|HG^QPq#(T^H|3x)vOD;F#=osdSL zMk0VBBIo&HazJ*7oJF@E$Hrjc6tqS`1Di6Ph02itm8uKN1*>veX^^5EJ$lj~E7nCc zsQ=`%501TZBt{#{aX5!ZOjY3n0$!N7F|*b`P3q^s{4Ar$JE#z6?Yhl3-EdcGYS#E` zMm_r2y;$H;W`$b?>e}kj?Z!8A@=i{Xof`}@uD$5?u|3Z%Zf%RTcIVL>pI%r+t+Cx` z`=@tTkqMffiHXOsf(RC^M)x4sR=v2fWJcJ!1__|BOMh&slfH z=7^uaPp1y;8b?Rkii+co#^)Cl)T>ty^YG=Os8r--C1Ud=jyY)2vKhv+ls`7N#K$M* z=H%j=S5QHLgPsgdH%>Zn9Pg+>!$z?DiYFE@A2|}YXZK!AvEwsY5je)WUY%o!C%^;< zL`O$~3>TQ7D}u#lM~@`r6cj`@jKBaH`gJ&8!KpI9$lDu~0%D)-HHxZN-~)%cmrc8(PubA2Zq!NOfD4|A zKPmt?@Tfc$v2#AXTu`~kvYacr%Ms%dmn2?!pG1X@9yRHYmFpTbig^0}9=hW45fIZU zm{Y2XX`RZaTojeWsfbrU1-oIx*8cr_1qBD;fHjDRmKHXlKJ?(zKmPD* zow~u#Jp0h)mybk?O4aTz7pdklj8DoMeAoL`mw)aP<9fDjRCPYJfkR=Dwd3{wsk){c zKfEK*&wdbj)wxwG4i(F|60e-)f#<0xtFBJPv?}^IQ$bKgBV4I$@GNV)lnUj1oNQh0 zIK$K{lDS+G%}^t7z7aT!vE=hDisr8oa0vnxs~J^!RIEZO`kYFDvdv;A|BBI_WaWa& zJ)SBR&11PJWx1UumaBrwC5I2+&3^3p zUzYqf=JF92U(&x>(E&r@-l%R&Jmx*BbY!_GkMIR?U zf)pA~A2=%2qM~O-pR!C`>n@wtXr4mDOtx;lkvraA(vcq z$6b?~L`VJY^1LdFQA4A_4JK*=BAz z0ZtQCj?yKM7dW{}Wv&h~R4Ul1JkL~pj6Ii1a$cN{q!7k@M{zo00pdE}rG{W^7S zJ9x;&IBWF`t7_K?Rbx20b&`!=`0mCe>2F~_kb!zB=TdYC)4h;RujClg;U%j@@!_VIGJ zp|r1l+x5tQmpauvH^;lPt{BiI`rP8`SQ-J1fJVT*5pcI^!@b{@)=ndEDgu9v>AR>(VrDJr&YbkR34ojjkvYlQpt zUZ7oc0~egml$~GIv*VmNYllWaBcKs*GX(S(h}`UjYISNU0_eA3X9zC$YtuX5D9$HP zJN2Qgy4+9Mp*@>V7}m+DvU@x5Wy~uEv~f!v>(>dWx?Xv@15a{z zeX&|Cjetg=W+L#=4Fhn7Rn3(BETj$#_Iv)$;b#${Mv(^l+dtw_6{&M{2c|0U>9`sJ zjeuJtpgS;b{d%=>8Uc+!O+x_ZeZBd>7;HGHX@Z}fysz9dGRPIJ{Ig5-LQ(ts{^iq+ z=awwi?44Ug9ZMsi5zq*@Jp#G|VbCwX%Ra~dO&d1a9 zbkdg5Kt{?+)GMJaY9#dQL%@o1-rnpl>(mCbNfUU z1yi3S11{OQ`M7LWE7Zi+cVAy`9IJd9<|+vH+`Rm}{CsP%)!W+}=NEfeEbzp2CUE88 zkg%+rVyiVIBp648{mtvq!KX#ws!Q7awmEj$hM3b*)yP0>aDVv5fi+U}vk@BeOBp!@ zXA?uyR$l~cHd|(9R&H*dhldA_OTiwz>MN_3Oe0V`5YQc%+VP%itCq)(C9Peve(Tm< z`}W0VW@h{Q`S$7ErB~k`y?S-?@;bT3L$%QX(s`fEVS=g#f>_UqH2 zLAXl&^!ezk`QLmtcVSje&a```TzCC7XAebQq zttP%oEweizpgS<`^ft9Fe?j2DfrAg;_r!(`JBn-v<*XU-Q({ zvz*lISd=}T*BRg<#~Jc3DJk8tWB1elel_hx<}Fhv^yt|gqkT@nRDJoy7mJoHTZy@m z*-zcyu5BBP11bW{y`YCS|MP|Kzx(OoBZ(y?W_D<7Hj{^W=k|Szm#hq%^Zx87W{w+o zc}1Oyii%!-`OO9MzD+!qVzri-EEWJ&VzsVYyT#LD>d>L(V^2QNr*AJDa8@yHS)bI@ z^!+hMG1-NYO8DyvzAQNB;P&(J#AcW)W`24OajR{Vnb-Gk9mVs$-Tu-$SYQG0mXlM` zrcZld%jSL2(Gl-|_(t2dt!vz?hvl=Clq4so9f&#X<>>)8RM7UV#=S=^u$v*EJ1}nc zLbWpN->re9uYcq>MgP`AbY`= zi?(muW3iZDe&tEjPcE#Pz(2pB;OC!z+qUh`kdVNCT{{-D8K-(lm@V-zdw6^LdivTt zEtQ{QzS=KSxixfX9<{COBl{R4b`{r&T@a`N-@ z3X2UX=>=0JPs_+ipEBh}C4g#PV8Pg=>uy{5+a|NOx5X3MTRlB-BD06hYRNAwHkvJ) zx9z+6rn_Hy@#)LP@XV-6CvX&EMx1hN@jzt7zeeBHKAyknv7LVn+cXKop=8f}vg-UN zbsLt9n1X@= zEN#UZ@xDGj_=4?O%i)R$JfhwL%VsoKOO2*~+%UL8CCx)4P{R?>9he&a0=4)R5J0V2 zSXlVOkH7q}ZW9{QjT_Z}>i=d98+M7+W-TZv`u6J|KYD*&hfZxKPo5}IjcIA=OMm`t z)r$2YA->PO@EDr#6WbX+wu**0YQNQo_+gv8++9#tgrL+^*xc4 z{OQMEELprF_HaU4TH%aokNWugUUEr)XjHKeqh^6hZ6&3cMZ%^dTeUtW!HK|W9Mz|V za?_}8E4Lh4yDRQA($6=4Tjpm)fSr_BNQMC;{y(djnph*C5jZ~x z=nl;J!BunpM-eD2C`>t#jNRm=R_lEa-Z^~uAhfGJEFKm=%QgSHvU&69cI{hZle)bs zMSI-rVf64ao3LfQl0EB+tgJ#6YfvYRGe}jv>TdS%FnM}fES^~K<=kgqmZ8%Pdh`G= z%*TJyIdKLlhuPC?ER|rH=m0I|ZPu>YFz2;*%@*$fKcB0|T{iu`+voz(dpf z4e0yK)35B=b0{;rVCH?Z7cQC`-K4Qn%E<=~yAKXWFre#l%B5x?bH;K4<@9kn&V=kt zXq*gdy5rS*E*o|KCn*_uHCmCUrVI|RTZ@i%sQL^H4%DvEZ(Nn}MvWq#dgif}zy02| zd#C88P0m3hY}`8*cidnyVzKKvM0ZIPS}qP384c#@Gq~Xr%`_8@KutxUX4}V_in^Bj zd?J9REi0N*W2u$(O~XbJk{ag*P@$ra(yMnjRkLz>Bqui`Ju^SQu(Z@#SX_McNL-!Z zKum*#goHTPDr972uHUdFG2sLn=HU(Mckk9IEc7qm8O+ZwNJ~#IDlEdpMOawfq~z2C zF^6zqK)ZIW5R`cAcusCEIwl2$g~yXp0)v9E^s9dTFmx@-!l0V=_IoM21PCqL+rery zh;8Ps7g0|lLnP3>^3t5Vq7p2x7&^4y9d}Q5_FkO)vGn4mDHGAJ^WZ87`*q+z?1BZ~ znmm1M#Rb=0|F4@SUl-u->j>np#cb}|x97}?W@nT&9izjAYPkH#e)iA%&tX4}>+A{#fXIFo}oXrt`icX0dm-Db0?b?fHM zn>A4fnjs0kS)PYxN>9(&_2<6BM~?dW`8R7C*}8QLZ!fR%>ABH!NTAOf_l}w{dqK_B zX+-Z9qx#jn%@0)pys!ucs7jZwmkVZtt0aMoB8S%x9e?c=D7j_%snHA=M%V$AYmBGh zjv=J7hEdA9F|1yg36{@bHi3iK0lUGWA*7weBaC*5%?4`->#8`YxMfo;=T>o*jRglX zk&7c*X6*Pod03^S;jnU$xXcW*N(D@Ck)hED1rWDfT6lEI15K?~8|)i~)S!{mU(qlk zB*A9Hk`h=_tXr$pPLfPt8iCr4fbPK5?)UGG+N1J-$;WOE)DT!@^~x)6wQSiuG_($M zRaFHDaG`ogNlAP8K7t3LkV z)3~_f8R^;Ji>ZpJ#tlYYG4lF}<2^krU;?%qHf(u&&U@*p86(FG9x`Om3(viQzCqov z&{to7Ha$Ik)_t>&CZrfmrenubpLyaXPcL&=c<@UvzZe!)2Vywi0DC3U{>Lw>X-^+d z73!pMIG_aUQ-ieU;fbdFjxB$hJuDI7VV7M#EFz*F2vnU12Wvq|1$S?6FZHN=eqXgN zB`Fg_iot<#Yy`t`nKNWV`$TIl8D|HF4nrcb{;An+99=;(Jb zBKOIF6)vS+a&pQS^S)ZOVogqV0e19a$|x)>q))$|Q}38ux9(pK^Vzg%+kamBUwlI1 z(Zk0<77-B^**M~^X;XUj?x9%W7}KAATKdhx?~fclmYkgGxxTU_qTKebieP1s4zOW#_Pv0Ixh78!SVcR#~{+N@UhcTkoEt_37 zdU&f=&F#G@^o!J#IO56%ekZrW!lJ!<_AmZ<#i4^oii--oe7!q%>Tpq?Zmrw2@bt9X za77GB5)+OsU%vX+Uso3w6vAS+Y1eY}=;19|G{s>pN-ErOdgQ8=YgVpUo1dEx1KPBC z)McZFbn4Urm0G2`I3>{zjX-TfKzCqj+Z%U(ePQLjeSKQAY8Dvmo1Is*a`}cy6QLMJdi40V?R!cLCR7#ZjBMPz+hQ^`j*3W4P6OrAUsuh3^j(;nGhpPvMv)ER>5S+gp%1z}myI6w?KeNArsrp8=2)$#T-lG+Zw8xmVk9kq z13s~_M;?FVnN=&-<>X@bYY7(8S*=#UxMRn@EgQGJ^TF%r!>Df6o;~{?d-xd~0PW}N z(X2^hY*@nfn;qNt9zJ|@)tbcs0cd2C&C@LyO?*ayV@lU`0 z_~S3rrr*}RN0&;^PK0T~e97KD2R?f53+y(umYU+@PrUuwyU7_DMFmz55A!lVpPzpG z^|=>k_vq1uT417E=DOiP|Ani3WNU7#+i&%&E8EG-2Y^n0kV--MV$>^UuAp zWz&w76X*b2N{pr0Iksf+(tBpyKK80HWo^3>K;Jb2wHX23fvL@J-)(hAqkq(>VVgH@ z{`iBhd;;ri+WhC5HTO1&j=cJsF*i-VzF|WyD#OrPXx%!GJaW&l5tn@Q-uyj#VtlDH~H9)99})P?v6=7RZ)UVQHL6B&7L&w0P!z>8Y8 zh*r&XL#YkRwATE-8Qq|sy}Jw=*w-h(uTg{gApw4`zWMaL&%WKfVQ0ezVRufO93D|0 z6=YCQK)K$6suGo_huL%SPrv{C(<<~*aA8UVceMU-1{gX_9#~j{2YVfA$j-|tv0}!? z64j(pXx$LIJhJ=@6G8i94*B`}V}%u#W>{?{S(#bw+BA=hY>4%2nK@agAZ^y-;Ufm3 z)5DZbaW#!@?C;}gDL_`~<#FX=z9KRQC?Agxr$ z8tpmfzR{!Z`y@NBum)){d33j)Z5!1ffn6eMTvBG$`S`fS5;E8po7GTkGhq#Ei4EnL zmaC}1hE-G+AD>S@``T=@4jp<)Xq}Mne^{E8Z;d~Z`o@3W?bf3Urj&z&LWf>5e9yka z`wtw#?!OkTnl_A#u$2@?G^mHdo|TpT;&cC*H*XPEW7%v)z4~-%(X7d;)tinUP5foa z>O(O{UVi;)Oy0t@RkRG^W3t*{V$F-bS)7@c>gVG(`KD_z!}9Ylt4^e5?%H$ko@tLP z`1Yg7h;Xd-`Vr@7)vk_+u}IhSnW;Nq@zy>RC&3&VgOdAe6G8wS`8@&ssC+ay=x}SgX z_1mxh&&$(?(;`g3Y%mqycGr|gAH5f&mDU0)ys@|(@;f7zb+FEs3*8a+NN`CC_=R^R zD#$n`lHnW>E|YQBj@|qA9kv#iwrLwVZ@~wR9o6}uLH(|~_SStdiF{fvz4~-(5K#|(nx>5- z@3?!)4L4oq?d^%dP9?1y-?0h)_cJu#u-^&Wia&Y?pGVFw8cI}FUq6@Ps3`8OUO-tF2pd+6b5J$iS?2eNQd z^NY{Vng99s2V)anf90Jw-fW8Tk&1H{G-fcFB~}Dwrlt4q*Zsv;o@v&+2~6A7o3}mw z*mK)=9yqZ7$h?miJTUVvT5DKPgwYPt5<>?JfpzTA?$Jjcz{0R$mkp^W8zb`z3hOth z|MClOB_<|+^x+qeJqAl#l#-JA;Rl}|KAMd2r^&aBfAZ;BzCPXs1%>nG&0oHJ&CHqi z`1<)`UECZ0d2jcg*hY;TPM>khgb7#q_;{nz={unB>_?y5wR7)#?|;^&ZL9F``Yz0+ znw3VNMkDZ-O?5R|6D_<(pd13hL4hy7`pj30=8hY8Y5UgAf`bFReEjz8Ju>;myWg7g zo@}{9ZB&X)pE#=v?eP+835N9G!_AMO9rg1(2%)-1Zg zRbz*Zy9a6p2><{<07*naRBHJ6t4EBxdc@V^F1;G=@s~zKhFXiP(*0ql0ueEEXR>%2 zP3FAZ{30v%4YP8^1sZ&LCa6wGU7SA}9$_+j7a5F2g+{-?AcT6OTjS%S7K39`sl=${ z24NsR)_DUQ1Y;Tp9Y6Tss)U-|I3&!CwzyCwZ{ED)*s)~vEbf{<6}=ZM9#iV08oqAg zHMYV$gV_UOI=f$+aana1@BGQO>)?s1GshT9 z)tUU`9B2dO@GfkP$Dh~+bOGS>z^0Kl?OP5WHV_>WScZ!)xv2L=J&Owq(cdY`=QJbM z?!v&rYQltah{%H7A{Mm%aTW_YM4x>0MX|LcAkg==J0=YpG9W0x59X&{Sm?CrQ~LJn zY&0AHShH!<<{jv1P=*Q@B*Bqlw$c(TMt$Pnk2Gx-C8Cyg@7DRQyKiYwKNN93|9nw; zMix03usjxOVE%;@%wVQ?UNX4xyC?hKy-4g79Y0eRMP#q0<QAT$af^TF6SH0YzF@TC_-J zK_~5}?|zy0`6AS&N;?k^PhWq03e$)qp0{kTckYP2u-mrn z%E`kSzNJQs$*ZzaVI>=*bEZcWH5hVp@{S%!C@d%n53hUK$iWCKtNY-={hr{Qi6!ZH zQgKmnojSq)nsC*z#1k7g?^yWF&%Z2P)v9&#PMzEK>)(6W@IgM_{H$km)2M64UygCj z!-o@Ied&J-zWA;H_8iBGNOy3U$(aq~uhIj^sp z3+zJicE!4&0~6x!<3f=xJS)}2w;L+e=(i8lKo+hv#aFF*^z2l>UYJw1@CM-+4YHP4 zadM;6Qw$o-D5r2}329=|($aogvD(wezg4T~ZrwXMZ7veUBH_VaR|)DlO^Rvn#gGp$7$Y&SlYj14&u5j_xa!;xeA_QxKKJ!&$RjKAi}z`#JKwL~`z z1Zq5M&)%5qtQ?EkJLyE~cZ-%_T>wUlcx<%MxG&}~wg44ldOE*Ai3jrzhrTodwH$%d zwxw!0Hd_6fjDWL}fD$nGEt@xO(JZ>}MZK*>#Xm1zg^B*JzWfFq7*qu838<9~Sk<78 zn31jmA}(9H;{x@TrDMAsmW2d%0UwD-In!jE6L zP{d}5LbC5gJ^lTC7k>5K`gL1(@7cd`!?qne_bgkwI__w~U3X6bLrke%d+pe$sL0i; z*RET;dFS@MIHG#Zy3Naf{r%=C*NwSiq`$vk#e^!8w8s0r`M{XVADCNE<%4?99}qp- zHoEb$uIG<`b>_Tv|1p<{SFg?~%bH90Ur;7k089JI9*EL4gjQw0m&-2l(MU0C3AlPsd}~B%G8mQ#AY~w$^Q$ z2L%T@EiH(>eY|}AeDGaiuFiE#r^{dzVo(vq1QYV}zgP$xl^B0KB_*k#u$VJMo}NX8 za)LIkAv!R{332fm8QGXAuV23|x+NzC5SKy(XQ|=q#MaWH6-!sHS+N?+0yx)$1;b_& z&iA(!7n>~j#xYNtE*D)tY6NO60=ffJYhS%PD-J7<)moU%$22`AK2S5EPC{kSuwgj9 z3B6{`rkwn|+^j4>z#u^aOB(R&sF_$z;G-pP{^$J->$cUc8}RQZ@9*2UN1Zw$sBYJ9 z-2Us*6$T3qzEn}UBwE%^IisJ%mK7&>Nv(rQ4}4J_!jt}|xEKZ!2XiNts=Dx10DL(> zdIzjT;fZ}U&096W`DVvYq z(L+;%eL{L;xrYxXc`TNU)YRmZRP-*OI{;UT;O$roib@UmG7SbIxyA#F;JbD2jER?H z$4+4WXZ0T&zMB7ie8P#>UwwPnr9)aaXH30gcqY&DJsjtbZQHhO8ynlUZQC|B&L$h% zPBxotY?~YX@BMtQ-}O8%=f%|YRCm|(RGo85sZMb-2_+NpN~j3Q3Uv?d7zp15lD0fQ zy>FH}4c*VzAdk((zEQ~Yz-(SL$%x$-~XvVK;BMfj$sLLb_OJCXbrqW*&N*jq4W-#c%D3Bd+UgW?>Qd zcOvuaZF{3`ItlPP{@a&2CB7d%8TW7Zp|U*U9T5#^qiN|Oe}eN%86PIqKM#8Z+|Uwu4l&CLcu18tFURXs^z+uMlXEgoy|*2D_Tvfs#WxKNf-4&{77hwXedGd5 z%1s_|yLn=K!%W|!)76iVGe+$jf2Vw6zbwd&w7*LH9JNLLKTSq{#^-u8FLL}YczFir zUp#KrXM!EY&EbA|edDl4J-sxaon`R4-t3&?zXx!O`L7hdn{0{ZPM9#z50W|(;j=OL zT1;cYS$NGOiD<+-TJ6z?X&HVRT4+9D!Sxn?+^|DJ+PE9yr*49!$|>MVa!Sih4Nvw1 z3lgR#h^C`5xP+DB4ft6cf1NM-J^CE^?v`@=SCnHd9f)+9hT|=eW>NRr_!0}u<7CIb zaU4CXZtR!n4}}X#tR;Z;(fchX&?s0-ii}tr9S-pAO|W(L1BMXzE>2Gg@KNIohIc!{ zz*(C3CzOwBMS|{sArZvTjnjpdk5CHl8W|FLq*N~h(5K9F`vT4nwe(Wr-v!4;j5cHQa|$kqtnAf24807 zz+npfZ%<1NFY9c+^xB~AcY`{`ao&q!daBQjHR7u$(suZq6}d{z2bt=(b@MtSa#-c~ zbQ;;W(&m2cAV{d7KEW3e=VO~JA4hv7{J8_~Ya0!YUEZT5*BQJ(WFX%C$b8WwN`i_gqcRhB5miG&BMUagd1&-uY4s6DY}bw@($msL#m9f+4pgzR`6)hZA+~fS@Yb8xSJl|T-W{;XsLvbHLC+;4 zRX!MuBmTQFU@Ey323sI_WiVVMgQ)jgxA}RY#K?%2fwzv=2Qy+9|48|p9D3_ z7<63U^;=#iq|QqOkW%vMy5k=3mzRP+4eEW}yWd>`NPw6%u&JMd`Cqfk->&srd~Azn z9yUw10}PuO8x=tWn}KRPS?T$aTj;bFv5~JQ3t@x=&>PSF1l%j^B_bUuzr26Y4f^EQ zqc5+~!r0kn`~3;iGVr1!S$OJi9ozPPOP&8~94)oKx2L2nH&02Eb=z+b6f*?9z1!C8 zeg5nYd)XKA`(d{6LI?it&t<>$xe{`{wAfryN>;Jy57$oe(dxC<-MRgK^D)_r<&GC2 zG6}LRVaW9Qc)Y!^(s@!+fPfnT4jP`T#{BH@;-%Awa}<+Zs7b`(Jb!;N5n7YTk7baO z{{5s!_(D^~ffYQI5e6#Js^Z2lg0qN#0lOZOkABjpif=mitfw({r(?a{AvzUDjg?jC z)vGs;TW`MY+i?WKAeDhoz>1@-smY$;*G1REbe>G=C?pi*#qXx)`mgT5cQ47J!s{1X zCdY*m7>~gsx*a2nF9{cu4X>ZD(`17EthP@*`Mj+o7c)N}b3dQAJ6~TV+(SMh;ZG9* zCr15AgWroi!vbP(&4S4haKrH!%9|oho-Y#zk|#B7e_d_99d+Lt_-!~BK!++JPpxz~ zA7m|%0f`f-xoYg@ivmm6UGDQc0oqGlMxQ`|+)s8hudmhZ7a=#JiY@P<(RjRD`$Ov` z!XDf8gYWsgq&)JB{{4{Ff37=)K>09k8BEP@E^SVT;)NL`i~Op;k0@KMY(jvl;{qos z;B#b++st+8C?Cffs9r_DnMOF+Oqn-;VIb5h_oF?L-)CFKUQW*UX*hD6*J16`(n8lm zcl~>j5z)o`$$I(6442{kDBd+bU&h}+rCyun9lW{G@zMD2eW&Z!y&oh@t07mxCLy@k zk7T_>jRZ_mAM2Rp!|_v{?Q#*bs8{R-nwZ*Ppmdy{+{>c`u*mymR+ zHQYRU3Unb&a`c^R`=fUxU*@yH#Ri}}R*QYE<~Eb=`*9&OE31nVxp`4dcwCm?plWS} z0vWJ9G@JF^NxL$}t|+15CnzpsE;lEOa6{qZQy z`dmT*+&;&zg@@K|Dodfndk;d!*a9(-_q{N-Kp}WWOjFzldyFSbc@GrY=eT>?E7| z-9=-a9me-0BDV0&%=~ZwYvcJ5r}TYu?`GRo(qU~h%QfK}r%7f8!ld9YJuW#4&I8kc zkf#~%iCI5P|0|fVhM(VJ>x)VnOiTGg7=Km)OAlUpCX+;cj<2SV6-CI~T-yN@q}_zS zl4pGm3Nj7z!2jHR{OEh{u!2dz8(kQ}UV*7bV>Wr|Kc`B#f2==n5;dp(jT9{2j%v3z z#`t=wM)ZILJ1k=u&QCppj(Q<<8&Mm9hzDkl=dccagy;R7iPK~qkjesx`zWHWY2an8G^fgmg@?}u_A zSd)>qDNG%Vo#cas8`9Ocx_LJkWg)Ypj91zCUZ7=)H#v}$j&kG?hh^g!xj`*{MS~$wT-#8qT zva@5!A}}I>h@oxo)%dIk>n+}&q&Y8=viBfLV zz#JBb>1cf?PbWcRi7uur56tq$;BPufx2t*BA@_Vf&m&Q=-zt6Q9O7{!wB*rmoAh?SpZI5Pr)8dlC1i%JAyNTuOZSyuH4H_d9HM*onc!Ng%Io zJJ~k?Rq5m0b#0&!H4$ME0f_+HKfUhH4tc%<;U@~YoZb(&*_43&5$$~*4ormR`#*-&knDB)aAFx%Qx=$Kbc=*1tzr< zBS|)8Il8RXSJS_ltX{>`vOkFWY} zJdLCTbPdQX*jk-fKFecSZ*;Wn%qpWh6m9k92IsVsslPa+W(>2bQw z%i0WD$JJpRb(|86MX~L?)J#E_NJ04KzScl8Kmrc^crH-H)$81C`&I}%X;+IEL(VD= zL#KUfv*~@vOy}u*&Bj?fNrH%kaKHX_nq%8Ae9J5i>7RA}y${OPGw{v+5!h;-IY-lu z4no099XwBu6$o0Z%hCu^UzT{Yg0LY0!w;zsx3NvP1qY*b-1oJ7`OI9=jr#=2d3gcg zx;1Dc7zH6P3}58w`IQhFN(={&6N5wR-H4>(UY;iH7+uY#o#?6dC-bz^VBln#{8qla z4zuipW4{@|RPy5jG2{UK;Q}!$g*+Kmwp7|3x?=4u^wy$}%2?8}yHDGWDvyxu+}fgB zh)_>Q``x4_3NSM>GdDXikmKmKt86IBrMX49pE2|2OS>oFA)Tz06t5rxCIq>3LzR&k zx*IU)^PHEb$7E1&W0Kk(SA&n7?tqk0u#d<3JN@ALn2MZgMT5CSA(2VCKOQqKb3Z;0 zECu%(!9015`Hwm#kmOWud6@X~D>V(?ulAUzOP3SzaOZ)Da!5IImH@_^%#eqkH5axUEoW^zdI6X`=^HRb&*t8n0$xX z1gBzCk?vhwS&HLNO@0H_*^d?ziTIMUfiWRojmAm6|uM3y5u(82K3LhOD7~kSOS(f;sX)>Y1r74vs5RiS8 zqEwj!VUY3+zPoMm`pY*3hDM?Z4k|UxTli%~X zJXh$5qE$XCxYf&4_z8+@wpF7OEJZ7_qzAJ?OboBs@6s}_3j-uIC&wAZ)yT*D-k(Dh zDRRtuWE*Lv`n(+n!`9#?y$u?rl#uRAu&6lQOH%{$=P@vip&@;i4z(hAVJ3Mcx880C zURs+4kB>eNZk>D9*T$~r;>FrxMm#$8FfIvW?z7VjlY(VBNsva=a3G9Lj>}SNvog@o zkc-FzUJsw2x9%Z&H9207Y_9@$7$2hYHc=r$GxPJ=aMTn+Yl;vGK@6+2siax@X>g*50$DntZ}85d`M`t>&(Z3XC_<IbPrGd=I(}dAHBz3dhyh*6_>1lKF=lO!Wf7uxZ zby)KS^)V>0)RlAC$~{Thf&e)6e-T+bE-431q8Z( zi!ro_->c)(%AS|0i(tUe$}H?c?!`RZ1edqT0fzi71T3NDK59{_Bmj8}RjLewVzo8E z4{9D*aE$vU4WRA$6xoUpXZgEMzDcdW{9m^+OB~LRoL+*QLcMs`jHbE#_mTva)x;CO ztJ_VABPk&Vk7!!(F3GO2L<<)mh>l+vp%Rpg_5uRsh${dqh#HcASxf}O7)+UeR2)l) zcYl5X`s0zO*-l5dpzCZ#MOT<%>8nn@$gVJrcncR|#>dP@7Dkob6bA+UK`AyK0Wj?3 z0Y%I-arvPW#_7|h#bVrGo2dS5#RM&}g*NH`C^khAq)r7DEjKtBnU(rJ=dRlZ%-RMW zw&Fd=gxDHW4P1S?$FvZVPxgnjnmJ14fT9OubIm`TGnE^>pxfPNE0uf+Y=TM>&8ncS zl|V*~virY$*%`JC;{S~QBzAIMwW2@y`*3WJC0x`5{s-agdnFt^%WZY+>Xj$V4%u5f z{6klfs}&|h`g{$0_nw8PNndxs9CNlFPik=_@z9wk3XnaRT>{QRsMM;BZr^DqH zEB~E;Iv{{p9cP%hoap~?fIz{xVJH49#m#AICa>lq#oY?-@3+h{)N55FKHMK!CfmF| z4xRPXwi9^307!xbu{^E!P&NPWW=Dt`U4@@qyt;hAgbr|gBq+>s&`{5`lclwsl{hEMP@6+t&M+vIAV^SEtz~s1}S+i6`82Zf8vHQ7ODdhW*%JOrSqfe{@Gf->m3wtfI1l7)w_h@HMNT zD_-y%x<6S_wfVnQBKhwHIQ~9b?{fTmq5f@Rr+ssMeV2BPpav|#Z?9a@D+xQ1UtJGI zemQXNJ(n?r|KAXPNc|haCx$H~@ZR9{gnqdt$q1-7O%%ungPPg07*_Sj6GwXO*wR#e zY8l{x30VGcArS-e{Qm#6DQ&tcpf8p^Aa0zWxvmH_FwjSl9E7TTmuj4x5<6lwFTL+; z-M_DlgxkLB(PbS~{O7ou|B4LuhVhv4|1$<3>`;%lJPd)y05I?SR`nbf(+^@m00Z!mm;y>2o7vC@3e^fN7U!A$5J%PeVC#1C7P-4E-v{0rQ`k|yXsI3R+f&iTKEWd$)w z#@uS@>Q~U*-M}vtcpSys=HfeAJp9mv|J=+VwUeC7xYo$`dD9fv%-tBiSuUefxV|&X)Y|*HvhImx&)YC*a=swO1<&0e zV|DaD8=S;O^b8F32tGCh|Cs?Y6G_3K_f%PqmDTOkMB>jNKOQC#3<}z8?am`td<(xG zp37SEVqQAtsm8Z`cfS;>W{ye!$BQjOsXF}cj5G(L>ko8HQ9BuP;M(lkHN6_O-^Ye~ z+gVGxVZ+P)KGBa0ffN@|)&Z$R{gq0weYm6~MbZD;Zf&+%Gna;yn*UqEc?@8y@TZMF zZ@Y%S#x`8-w{;hNWn|~_``nI?`|OlvCSB=8nPD9gs6q##0-?lcK5s+b%#wzr0xr42 zXc5m|&FByLY!`($eQ9DdJ#7E2;our#U@XxPQ5yT-L|g^+e<=`Lof>EJwrBEDyV?-k z!NvsY^aU=|^w@e3rXxnH_+XuB+L&1$no)^|2`TvWP{~pyRBFsech8N`4_}^Ls1`sQV8=1-N9|Q)#P5W19A2!ww z5>j*edzJ!aDazLjA!W;i73&Vn2f`W#Z4WY;=$aZ0P4ag{r3-RUA0DS23w17^KZqAA+$9Bqyq%KAWI&7&@BYOJM0+4gt~(56$csl4PC8x8q#~i}WsJHm>clp2 zoa)ialA5Gy?8Pckl&}n%V2vAlaRAwe)_4b<^^k!?bcft6A|;XRG!>s56eDz!cH$yB zr7|ot-s)IN*Toq97WFD7c>|{gq^#&p+-05>#f99}S-SZ~r6_nj7)cu4QOXZxdSHg6 zK}|e~&x_aU0)Jc&ESpgclVo^=%9qr`J5AjqI(%7#WRe!+qGn=>;z#Ni0nB(T%F#`- z^g(&UdISiot8sMOxTc;7bsG!38&nXsiVP-9E`hLqkB?p;y7&P;`A&$3>bt@i6_Z4gi<&>e~Swo5`mo3#PrnkM52s5pCA3E z9Vg_AZ+PSjCKev*p6fIv5}7aML8t+|1ameGdO2qH4zfo%MLC6h3ik#`g5tkjKY8LD9H>pog(@rCp*swmA4{r4wfUqMfDDZ}|)De4^~dKYv>QpUPG z0n7>PNhhD=2rLK{PLXWJ7wL8Q;8dByhvMAKJ;+GrRDcKg`HdPVmGx_#7Yz#b96w#i8C8)#nv)PbvrwEjuQEW1=1*lJktgC1_qf4}qLrQlymAG;^CgQLZ!bnc~ zS5g$WQcUdM*!Ogkr{~M*xhNGI)@QDPBNuVnNeS|cL|H%uQkK zoJ@oQbUL9qTJ-Dm@PxPwX*@_bq;HIR*eyI19=X)yUWIIw3^>CaDQ`F^Q`=lh=OpXBMZeaLKy@)M=sb+l6H zxHIE?PZy<=PU3M1UI12*@CF#-LQrOVMi;*FGX={OJ+|TzFap5HOFWGKJ!eeuVq;X@?#va7IqWoiINvC4dkG@Kd1g z8^}i=pz7=;adoI^iIebrrfrGFt_Rn1z}AEiToWiE2K6U7V_0rrn2yVMcxkN^a2jiV&75P^ku4URm~-vvX-d9+!G8|4QJq^zYO(mrg` z28-eb7NyuC({FCVsKbpd!^|_Ndk?+Ns>eXeg44_AC+__eijwUN(VCuS!pCu8`vK)qEul! z3T|TYin3*3gig`F?BGHD^hPdR8(kr)I%4A9WyoOWtB66Nh4`uA#BgOzwe&Rpg0+ZZ zi+F*Jii_2u;iQzedd{EGNq{5LWMnP8hH)9Nr31^s22p7R2Rw~)nn`*b=sbi)n8Ge9sjMkaL;RL8?nh3H z$H@psMJ+N!Fu+er*PF_9$`B0D!dOXes8T4t(^g$Gf;IzmK+YE zg$!|oxk^bfpq)&6;yZ*3jjF%~b#?E`yNw}Iy3{E|C&s5Crah+N{19qpW`9AkAqHzq z@J;d7iiw-zNXS57Y@v3SnGsL*OAQ-Mf#akNdHB`VZI>Aiu}@&1_A>_I>tw-%!bnJb zg-uC>A+%ufVM~jkroW_NFAVgXF}qTyS*Ik7$jO7ZnL*MoHmyLcjfW=_Gee@|;KQiO zoW-O-k(vY>7MED>Xh)V&ccMZ7nq{=Y_W&J55!mC6>ftG*tmJGWc9TKF$diVnhq=Vq zFH!>GxQuVyVKEBdiiOeaNI3V{IEDwXVbF@Q(%{0ZWEOI5nDdIIAw>r!9!79u&Qh=m zt9i&`#nbWts+MnVQZoI(UV zp*s18xNwP9lH_yDRIE&Eu1P%|jXuhP)4H_2ynb2RyKpW2voO_Jcq{qC(}fowz+OOD z3XrpZlT$jA9+>I!{;JHG^lD3T`-Z+k!-lHDUD10@|EjsTgHiO721J$72O$a2P$FP( zkicAmQU%~dQ^uGhL*nbHz*|!(V?=Y)o8FiW<=6m^$`dJ6u|&KulxzVcRXb9F(Xi1~ zNFnOQP=;?5Dj*5i$fIg1{c_Z|nABK|H71$Zin?HW@Q~kn={oQm@fN(9ogt)W&eC$K z5KZTIfNkFJg7HdJrC@0$S=@i4(jGR#M*&!A$=@k3U?PLk;PWJ)RFWmr2r!Vdt;9F$zz||wiK)TBYv;NKF`Bg%Y9UWjGh|((p z5M}pY5rr^WVoQaCwdc(KTiPaiWn3vyl;k&6QEA5Ut8&_WdSE(iI;AL8H2$3;0&KxL zqzr@%2C+<3z?vAA`ikq=1+bDv{RQzbqodYdpiJ zYRp2)vtbrfVxK{)E8e!0R1nh0UDO(!l~|O%Tvff8$!-?rgc9E;3hzY??_WMlvfiii$TtS(}QP+nq?1``#GFqEBkhdP3SM}w^9dn}S7c0CMh;*!EwLI%SRkQyg$ z&(zK$sku?7!09GoAP3a~z=Pd_`l8D5-VH%ilPJV;zlh=Lxvo>1U!Y&a$c52Yu%uVx&xCde&>O zSq?^NMc{$dOs&jh;3INeHrkbqPGyR>QuYL0GvGJFX)Y}xrJt4xl4wR%*)RPtXJcjZ zf+?z_Apb&DKqsJ-Bq`DUi&Ljz44bM}x5o*a#s_eW+ zb|1TJNw59%R~sD_YMSN@)qz!)L{$}x83o_WySpmtopdB!CW+usOClymO)Q2S6e6QD z_IH1*Y*UAd%@OBcRp>G7aB6x8afs?r6t+C4R6%0M##mWHepQ75abk}&v~jYX1?|BB z?I-lt@v3*G9W}ignQH2K*kkap)Lk^m$_&x5X6i+jf}hyo%Y(_Ia`21;XokosngRXk z3p702Oz%;_Y+3-gCiDY&XiQ&XoNv(_NYnBnpHBm7Q0G$wD9YKiJVNKv20qp(KQG6L zQzXpHv09p!O;U`reYMY<)92nPUI7{5=1>|PjVAs>b#xTqh;%@GGHLoF=5>52Nq4hA zcx{%s$(Oh~r=+dcg^*i@iPS-4iMSQ2V?T1OeG7?uB8k!LvID!n)F@WW+So6KP=WLp zKTDy*oJK`?B$GjUkfXns@S~LUhPpEPy{D^B8ZV6^YV2KogF)d~*?bBkzM7x&vW zJ&;Jl;_8EdkKmrh$e4DP%5anzO6o|*gr%(v*|40NMAND+!*G2qF-O5Wfg|^_OGjb_ zlVIRH4p(9SRo8Zk=PE-kO+($BGTQM?#v)NA$#XO?s11^ijz+jOibPcM3zv?}3c=1J zuQdYTw*tUX#Lc-PCP)}dR0!X~TXBe;S!78~t>$rQT(Z73rx?hd5oYwQ z2Qyp{))eZRnZZy>i#cRPesV#`4Jk-^<#yDl`PC4391rS&MwF~h;dttzh+t&BEXIj3 zQMGO`w0kTn-5O@#NL@OjdQY*)Wjk0i(INUcVaizVExt`Kqy9mNdgOO&Th#iGokg_m zz366GB6R&TaRWXHykr1sxoA>A2lGyt6s8J4n4)-pVK@9C@#I~~3-9U9xske=vx_CU zcgBxk{&LnN9n5+XlwXHtV_1XiDCr0C@eN=+KoxwfweS%bk{A$AJ?XIS*%bTBb`dw>cHwkIe=EWMc)bAtHS<{1@VL|$^f#H^Kzuh$WPhR z()AJ~H%bbV)G!v|h&w6t?QUD}0u? z<){fp7?D@=!9p;9zu(~WIB{5`>hAAIbVdm>9LOpCWH$S8eOg}x6lS&)Ge<9QW%{Wy=j_El|)-kPGcIa!SSo z|7>y@4r^3-`b}HlFN+p6&a%!%{UQ%>A!EHjaG)2U z|C9%Xc-RAzYc?a2lr3)!QZ`zcLqQ4)PN)$6)u-akPcy)sSS*WIWo6wY8n#i?S|&`A zm2`KIFViH`5A4@9zg(8P$rD3{)Lo4OS#dERsmN(KE{d{;<+86zm{#~#wkxDJZVIkl zp|uj*ZjqTm62pj~2g`UC0D*@j9m^n(9E{;?DhiO&!v#Za)CHrAfl55-8Lurvin?^* z9>F9&866aN$0EbQgQ`P1$MBNk z5pl3GaS2smR{V|9D>A$Vd^$~;C4F{!r*>305af_A8(d~%Ip43)<6iYmx$WL8%!H+kZi;LY!WOT+${&IGLG^tumqTvXn_vO^vFc8iMp;CX)3* zAr}}216R))ekOqGM|mhMxyFQ&VvlLf5dnO3_%G@F=h|Yp|J*cJ^DiR|wgz9!y z~7NL@0>JTtVK9Ln_P8|{rBhcMsVhUm- z?jaDD1(RBJb@+A(_4G-;MbR^L98wTPMK`1UUiD1n=re$?@obuKbs3~%*<6^_ecCIi zWikxvd`&PPtzDEb-hMwMeiF<$-Xa_18&M!Od1#W88n??n5w{UYq$&497&KbQZ)tIT zd4yPyweP0zTMP2&EeI$&rO5*CFNGm;?31ugRx2-YZdRblZ`{77gMvij3t9}~{T3r< zzHY|kWW}x0u9)1!j=Q4mW*)&zQ>-%(tnlF3BNyI?I55!r}`d88t z-}i;0CpV-e2PH)>Ks~_Bk- zAJh0otQX6KmOi)y@J=BSB@=lsD*w)+K2DwX8{(zt2G&+Qm`N2v5ix!ITU>ZEaSE?< zTI+7*x}sS~YpRi>BFgv49CgQdZh3&seHAL9CJt?S43+ag25c~{Mo12X*TDBQKYd|Wi;C*&u% z!C$QxBBPRME`utVB9rAR>>P*%C?tG%7-Dv5WUE{U_{w%9lO^_c1cjli??X=?f8#m> zG~)HNSGu0I)*k(PZD(HbM_+MF`RDR%zdu4Uk9o;+Q)NXHDKWoGvx?7e?VAJ>AP{U$ z??dE@cdL9~unx#-AS!>v^P)v670-pIvWkyT;-{w#qtcNjB=Ls=+pxs#f)#uM7bdO# zyJ1{t@x1}yF|^d*s?H&g2?QF* z2KG?C`FS?Mo5`RA&qotw(LF0HK1e*##}CVj8`p8XtgW}^JQ^#+gwCs661XjYpW(t3 zyUvj+*j()UqiBiHEJQPw2jw;^mP>J68HzAQf#VZS^Aiwwu`m8*`A5Y$8X*X$Nwa@GHYhHw5wd8tmgLA z?iRhH%!--3I$CS++Mkdo4V+kfD0Z%fV=mJ?)XD$Gcq3g5QbGZu+m=|_|MbZiZC}#E z5eWpMM8ZeJExXgkthyV$wJx=!fKtd9c35%@EIkvpkrb%mE4W{(YD(v5;q2|127c=3 zmt4cF2~jbr4j?8`FN~#A&Q!EeTVII$4qi4VSNpim|9F`WkYAd|V|?RbSh`8cja7VZ zT!Z0cDNX*RB3<4i??njAZf41g7NrMGEkxW{^|~ifXV1c?ma6pC3y|Sa0Bj3=L1lKE zLO(ToV=#B}`CVr_ONptO@5mU3HsPPEP$Tz^^6@#br_uUWDH=JKX9iHb9R5OgN1m+G zh)Q}h@&N>Xt^DCrz)nVgc{y&{pE6CS=ATupl5yKS2(bD7e_8+~#>pW&!i3Yi{!`#I zrGKm--kiMCD9=lb)#DIk4?SMP$-^6}N{FCGlWkyR^>~N8hVg%sgUVm2Ojt;9h7tGB zL%SJ!o8jY2odqH45SWfdn7<5oCE4<4cu6HwG@q=U6&cZJpC!s1#_gch$o3dT*2crN z9sGg%rKhE(sRTB|1s3TgG37?o6yFZL)fT_8LamuUYK}i62Z<9`^pqn^ZfDfZ8+hx2 z-pl>@1Iv{YO@qCVuThRy4Zk+~gpJ=aS0|u-Epz{#)2PRA>Eh|wyH@c-O4LX*5Cwrk z?Ai+E7FVjL^Vq_K5(yHMH8;Ykrd9;8YI`Sj}!N4qJHwEU}7Q$u- zzl>D42Ib5=CDnw_`H35y;cqFg#{CS~@KG^OnbU`UXp5H%k($tkM1xWp-Oz=o@lngr zS7YIbCmSok33CbwehEff8{DKYl`DReGU9R&K+t4P*cB@p)DQOe`-3&X8PgVtyAbF} zQIy&aH0!1$3jGx#y{i^olp~hYhE#EHSD@1lwgOn9H#|}NkGl`sV9M+y%RAsI?=v$Cm=0<9M2wTi_|BJ6i5_B43!E)SL z;#Iz;woJMkuRWhDea-V@BfA+Dk*K`gq-7XQ>5}x9;J6oSPj}Yski**Bfim39I!{!~ zYa6C%j-~r&qUYSKtu?d+TH#J9O zMZV%+m^dlFLVx$WW=AlLgdpsNP2dl7770TYRQ}$RLhGTLX~HfX$gc`Uuy>OJ-C&gE z3L-6;dV7n5YWYsfCvda~Y6c|u$bh(%UmloUqIsEIwbPL_pq+htJ<|@l*Ys)^?$RRg z`@12bKM(VZN58S&vff$=MH+CRSQW(UgIvftsJii;j!?2g1SD`{5~2+PP(3WaYZj(1zT=5O?z+2Uq39D>l4B z-bW|%!WD*c_O@%Zl*I}qEjtu3xfc+C1kz+K8OQ`63 zCUqfQjCQa_^AnFxSg+B?eX_VPXGc|7fBx`TJY~;jukbOl z{JqPVUoFA&`A{#6Kj*V5#NotkNukK}2PB`KlLgfA=Od|$vJ}!}QcWKHgzpp2s5@nO zq$HHr4YdX(Z?>}E7M>TMpW(NlTBkxD*&LVn65)F=(}H1$V-Akj8&b%vrNFH<OjmoPK36Q0tONP*3FdxdJcfFZaAhNcRRkts`WV4{zKQI?8xNW0L$*KSq7cY2V zl?R1TrRk!L;-Q&a489UzvQJv07hP*Z$LYlTd1M<9Q_+ZYIp@n4_cnJkdh%n2i7 zfp@Au;?s=ZuTaO7ja_!G<9luX;aJ9h??x`bfAm@2zg^C2o~6F`3Wsxbl(BmNL41XA zGI`J0Lo)?{kr0qnpZhqKc1Cr+^v@*tb)JU6_UwkMpu-)|d_+OG4T0CjT+xT2$i_Cy z^b=!){6?NffvjP4Bxoa9^{ft%kL306bYHbnpCpkVVvS1hVK6hv==OQC*wsLF5k<4` zv~;)!xTRqqvpl0D=bi9Hpv~_(fw#@cmhm8w7?Kv3D1eZgDw5YLoo=I3`JI>=89s(A zyb31@Gti#AhlJN{i%}>TOkSwYbClM%<_olVukJ`x_x@TcZo8VU5h`^*V~aeRKh`7r zravcNRk;p1JjzC6*w7`Ty(>6YdzA}?F&~^?8IF|iJy%i@}8iXm``E?B5A@HP5x3WIwHp2%|6U{80ht&bd zP0}FazX{`Gf;ZW9^G!vO1RsmwWt(%*ERvNz*f)+vPB(U|a@Cn(P(n|ft7gtH%R#WT z5Z^o&@y8^kN^rx3UzU|LtJze66RW9j$fD%$kG_f0EL|nf8;9z7XiQjj&K4r3PTADv z{Ym!e@gmp;5|}d-un~wypLfK_BfMF7@rBfzXLR*b9CiBqwW{C6+52G4_U?rk>w0QU zAH}L*KLY3G)v!%Ycz=A?p&GFnkMvDa(W*crv}K4togZ!tLUB;EG?MZ>ja$%}DqOzN zWKbEan3!G)i7exQn!8srC&cGSr>BtoqTAtb}Kn#R5w& zQWfUOk*NzMT4M)%-@C!zL&b!wOedU#Xem1c5n#gi2E6X{p!9d{$dMA(QjO9RgpIbw zoT91ghdT_Atn$bybJWK1NPExk^6n!)f)pd9mVZvW9(Md3jv68dzv8%MAn16$E26tQ zGB>RhFGX|5&4=2@j$=K3aDPiu z;X``rrU^3tzL7;IQ6y$au?gGlgv^Jci(Q~$sAFoV!Z-NJ4!fg?dMs8=akqsqQcjkX z4@rphlx5|wbipskkPX2b#b=|sptZP}O!LzwA%q_fTnn;m)s=sqf3#&H{C+1|da_A- z(RNjYKT1XmxdLu0^{brwE@lQyFr=bM@w`|kP`;a)C=?n~^t#pI znig+dS|u}p&J=+JHW3+cv1!a7JR@!l!q-urlNxN3?O|RuwE8! z+!=*aO=+ec0?+>b=q{}wOU10*B5r8G{&M>E*F9k2PT+xUQr+ln&6rO1`{_9*Jl{9W zY|6X-ccv*eZOKa{gWV`;?jsj%CZHezyy^<)woY0-#}m&wpWrQgN70S|%d$AlU=dFEZHE13(EN^rZwzu?GRHmm2GY2mQ+B+_uVVy}^Wry) ztcg_5jZP)wN%Oc1eM6awFUf;w7tX850vgGnNEW5q$||}I>qqIF&!`S^4j^G7c)&R; z>H*X<+93^H6WBO}IPA}`dE~e&gwb8kL4wYP+cu(YAEfJ@tq4y~r^Zg0T{_W)1kfXE z#3ywlm;vb9>}}h}*8Rzc2v7IZ;fU7_J^mfv%J;nmn)|syYfcZKaw}6Wt9j82wE(Ao9S|;O!Id7+PFvzdhR*?@5zeP29h~Zye zJ}7%gx|^*MProZVCeiQlhdxPg52GBDcgINK_?#IdDWFy|O9>roxP-ds@RI}k*VHF- zzi-x(CW(c`8Z!+7rOo~^PRKFa9^zmo2#s;x=M-3XTX;#4Id}Ke-K7U;7NXr*kcjZ# zT_#W?;Wq`$xso}OJgc}hw4QJpW9R*@fTftlYCkbOsu5tZ=F4YcE*sp}8HBIPob;QY zF_K#%!Tz_e@o`i?SB=l~_FWfN0Z0){I@D-tdZwWMQ^zk>o27|+`9H`McorJ-+o}=u z@${dm;@oU5n<3clL6(d-)s79UX`g&1$iz(K>wG(YMJ&Jxa*Uj9KF?b?qmUOB1`p!a zI6XN|{=VOdmyx(4(0$AGLW-xo_rW4}PKriAIhgF!f4y8X-y zKb6Qkb7axAL^#1tzTEzn-vkLgCQ~E`pXY=!=PswT3xbAp7A zO2cqFJ6Po6AIx&>GV(?N>8+#G(PRa0H8L;czfCG!;9*6m{bq*H_M2?gf{S9`Fn{m8 z53@cly#chD3dIlaYuP5@u+<@UU(Xy%`1G1cA7xMGsQL7`*V7N3&2TQvf(&Fl&;d8- zOg)*JWo}9N%Gq&M&CgxF$1Y%+$XT0$8IwBM-E^6>#o?2;tpi(Pn)he2x6R->I{3mf ztGpsQ965UIy}77y&Uls_U%T7Tf@FB}mun^b+w0< z)zyl4z!SNl03;FJGq13k;=Fs3MQ`$s8Fn}j80FK{w~Ho7qP7ppL=>J`-eC=Gc9d2+eADWy_#gKoIw*{Af~w(80z$ zBEg&q68sWL0)Yy<|Hx2n1`iCsxAK*-z}%zE%LO=%!?MPfNuDc+D)y88`j1;|OXc}b zp}~K;{#meaxy6Nqa2V*#4@ZJ)ZYZQhC2z}vi>di1=3B0UDkA3JV#9ADml76`^Y(48 zV^uB`kCu@mj6zN?K$k`vieR*23q&X^DsAJ7W@6QYNJ`^HB@wGn$|+DkY?+iXPCW1d zvCPVlW&kndnjzdw)Dl}zMs`B3+rR`wI@$yqANplrsU-LB3+j`nq6XZ8d1y5=?6Yy)gD!&C0F$}nsgD2GWrD97q~n`a?`HMd+DtW$Ns&} z{r(t;gfv>MIc1KrpbZ55T&b{U>L@;fmivMbWZ4JXgb3GrYI_qi(J z@BDW7YFJ>NpQ?|ST`a8i@?p+bSOsvt&F8qx{o5gZoOqYW!{bBi8e+rq^YTR`tcCBt z_u)R~&4=xD7vlw_AOD;XJg_8`Owrz6D7+UzAPKfCW7iO`$Xc@HBK#%5`Szhm0lJjr zQfXjZt2MI7bhjns_Gsu#W-);jhMnYqDFQAFV1!n%rJX*mo>|0by=AKQ{Z%BRM61?@ zqB~Oy<|V^A#wVGgKh`5D5wSpQ0LN9)PMsM(V z_Rq)6^Y&Tv>W&au5;rBid(+0<}wM%(1E zwwLvM#_w216Ilt3l}yR`Q|rXj--t;*8jZ839#h;;4A&fw$F~R&b&LkHDLJOK$1;@U zODgfZ9%4eQA3YPuiCoTXOgwU=0Yv20e7&buLn(z++$O|M+m6(W z^SBo%3(W8X#{u09#ex2zbQ^NkvKV*GFnmEH%z!fC5rji1=0FXXZ7)!sI0GE`A(QgC z-wH3R;YR>zuH4K1s#B6e3gU!6fPw5%tu`?ZH8PD1@-k+!BZ(SCinFo6Gjey?Gw`6~ zs+DXPjnP8vk;9fEQY;tnv#moL>%by;vK*1OmP_SZp-~)T_$tQFsv4hyT<3kx6K9B9@=tMV&SR? zJX?AqK}?D=#$tGv?YI!qu|kGRDO6L$Qe@%T$?``$6 z%Xz=~x(mpl&S=L4rMkLvdD4(!{GyXIFN%@&%kaUCFn)Jm{I+0EIzrf{18z0N>Os~s z;h4Q0zA6@Q`I;jvf^3BcVQ2>XGd-9MSy2tF3&d7=-7muG6v!w-CpKaqBWKpv~Bz?l{tf3EB(Um#hLp#qb# z2r!^=4{MO){|aVUC+lJ)m~`VV%UZl(a)b zG>J~KQ5BMdU5c!>zv~S^_6}P37PvK=v*lgMmV$Bs6iEzecKXc7jm zJ)XCZoNaqAg(T4AU9PJ(Eut9Lgb7K~n^Gy*^@-%f+}tV=GSODLoFk6)PZ0`wMGqM6 zI})&}p|Jb_j7&w6&x-!J>ZbC0sHCciw~WjHK80k>M9c)6NI}R9b3nG3v2T2!nRm7u z+zyXVUV;CAie^$MFS;@+8D@OvQ|@ zvFI={bUo!pCf#8?<{0Q^s#+RmoC!pN({Y2LkKxIB>d+3xN|Nu)@+0zUr$B#Z0|}F! zCQg&q46cMcMjH?Gh>ZxufH%Ckdks#&y2rWmpft`9WX{x)*PdQvTjD$tZ^!R{3z+ge zqoa#tS>Hz&=YBlI$Y9D?Hp)+&&0QY1ZQuf`<50flQ`W*IggM`qV*b0tgMt*nJ-waA z=-M0N^Y3UTjwC+NvgXprgK`T4fLrJ|^28-LdgFf=>Kbhtf>^Zlz?c;|sp_u+mHN_2 z014j{Yj}VUWQz05fD zdV*YH$HX#1j{c~Vzz@4gp`LQgX{p{M1829!s;(FU!wwFQ0wiDG^mztQ0k@JU;nqUv zKtDS=4=^~wWt)JZ2b}fl422*My+<0I=PLX}YJ8FtDWUFlWG2Pi;j3l=Zs2{V`AuHr zF`>%i9_Np_j6gjXd&aq}Em)oel?OU;O*@bJb?dd=E+Ir9a8PP<;daTv!rF_I)&L*e zTz0faC^v4m0u$K)V0NhMsCEQU@d0UI?Hq8^KW%WTr8x}UqaS?4!$=N@517FTjY5OA zNyACJ?Ksd1njONk8)X~#1M3`mTXpt zpfK4^#8M$u$B|4HW@fdpH~yg2Te0b*;Mr#KMiACuzsTWx>l2tMMoC`~xiV{IiyVN$ zXNXxXSDUq1@Sp?95}Bjs@OEI`DtV)8`a{tZtf$lreTM1)4?O|mI*a2S>Cp)sdTV)i zA+A#oCkD{Vgjnv4UY~O<8g%lYpr~2bj)oGVL+A%P%GUiO*loy};An!ICIb#+5q*eJ zIhtDpCqe+o9_gV>g_rOr*jnJKoJBr`pApD%LQW~7qJ9Qq9~6>orpig*3AL_9b~!LM zv(+|*ak?u#)FzyxLpL9@1ssH0X6~BG}w)S($8kO8=7Y@rX zJZx0vdu?3vrBIXsHkuoL>~ZV)-;g*T-g_V51wD`@20>9)un1M;2Ok85wI`eBr5JX* z<4%O&?v1GEglDC(C(!DB09!W=iKLu<0FVpnx`x-Mrr`$vKTxfkHg#HD`Nq!!Y?jPISVJ^tLOXMUsXMq z0&AqA;|0{&NbkA$U~9=T%Ov#{d-KCNuA3Th?C}R_t$~NjoOPcJ=dDpGzhU>v3nBso zcksuM9QBlm_&l0(_(o}QAyQ3r4(`NzLXpJs6@Ia=^o#sMe&<3F^~Uj zjN2?kf!KUj)R|GAMPhKvNC&uyi%9kjXM*TL6qPVK6Lh+bO&!PjHditk?E*N~yQ`%G zy*rFVanb5XI4n!Gp_L&r%o2w zOMRfw-$_@G7fQW=7}m94q-Va)+7t1PpA3z@5IDiK?x(}VL zz@V@UYP_61#2wD^;3HKKSWnI>=`b$NGLb+yyM5A6~%6VRF*>6DqT$`Ujc=|iJgb!tI%>566`yutZ zMBtc+xT)~~K>~d+bNjJbNQz#b%BQQ9jM^pm!7&^;ydY^{(&|!oDlprH?54|DPK6{E z5Du{MQhOd$8l&NEDb%brB@q-TWPqu=Ke`Q!g5xZs7pA^D`-I_$h}^(GPAUnN%uz7( z(1^@vHx}aEd>x7Gl=(m2)xr?R58LwJJ<$-g83%6z72k~O85ZU^H795TgMEE+^_ZC) zuR_eAo~{NSsUkUI8xA;a-Q=#%y;w$4>>{4}Oykgl=#aglWwKm2;ldCY4CPoxwny;E z9VyTE`Olhgp;Y#TCyY*IyciL+xAIlAK+fSt$Vy)F8DE%7T#V*sUfk$n1Om6>H^aI# z#h2s_H#oXN5yY0?j(c4wjMVlunDXZ%5|*!6falgojm20Qz-XI{Ff{iXjRMJ;REw;W zp}vKQ&W+z^s-qn(#mEItRZ6Z4-^x3)s2%N2f@^4ZpT(Uh@o9@PAz>0V8D_IJb%ZE@ zk*IVS+`1l>Psn8G5#;RF>mZRh6=oWGEeSqCMRxt4-mp0Y=oDY>tZPhN1|T~F5-0UD zYN9gQK*=}>H%LyR>U-WnR*3Pctjz*-t#g-?5TPRRR5jvw6BhKoGh;No)|#W7a%=~8 z69S)vl}_6;s(*piKY~ z#nVC;INZI|fq3D2&Q~Ki*TzxiYKtgIsf@R)8ls##cD3?JNdhJweZkZP%p5(4=j$)x zcs|S7CyFETkE7R&PqNkRPa3;hVo2c)z-C(YM4;7;n;HF)7EA6O}mV)cHIv%ZG| zu@}p;kGr$6ic3}?7zUAGhr}mj@22J0SN{vOtDUfS@o1HxlL)qy#uhP zzeerdj$Kbmo&_fd&}9<;7}q!MeR1W~!jOKQB}~i>DOFGA{+;K?_%fB zySt+a_$54&CuiY#N`VbT_}OJJ9RpSFI!gtEjgR8E=&;+VhZG=~`aWxtHhf_QXQfK% z+I|Kv-Lb3xzAu*~HD!Q)=HycU7zP_g31?EBs;GrIfk$kKlq%8?VD=24`mx*ersSvh z-f)KdOa?Ay;?}3?SkS54_2l@t`%UkEV0Jcg;}PUI18A-6xumpU0uIF*iR0EU;~ygm z&Pmk}hv0(I^fJxuybzP_ZaQ`dqC=33hk8c3_e z%Meo{I_9d2yKxG&8Qch#(~D(Q(VSdJnuLO$e%V7U3^}EjuP2q} z53=SVcCJ3Sb(k&l9;wHXM*r>b{cQoWz9jRZnYXko&|8zrzqQ`E4YzdLuAUWsxv!M? z6-yqT&*`a~yOp=Bu)MUJC6*KZC#l_a4p^2wC(q*qMh>g55wyEKp1tT5> z2}k%;#n;Y11IQ`cDCi~o0=Ia)mJU*#$rB4Ha7z*GsHSE_gIf0N*U#=trI*}Mef@aB zQZ4wvrM9niW!Xxp9fM92FR7}yxFoWt?(j_GZq}F-x^9*UXyRbSo59m72?MM0_=IUi ze7e_%e4n(mvDtEt#IdiNO{Oced35kw-A3t`YA>-*M|M23A9@pU4ilB>5;=2#Bc|_t z6h)7-yWpHhoyI}Em9LBiIBoG@zmj9*KsiY6aekW>=E?Q>_B7w&VF3ABqgChO{J&Zt zhceF(ysP_yJXhQ;y@x(T*jH><1=)5W*rQN$L4)40&F8Tz>9jMhJ{c}p2}utTKe3KR zF794HJ1miiBe;sdC~$6*bILZ*t}Q`Anhk~#)d3^!%~A__cP&nV$WBD^h+iz|CIhGQ zl1bfVJjB@uox})OPA0@DORh*2kKvazYN91t4_DgAj_dZzFpRKIenvFF?fP~{W5P0W z=BmVtQoAkEG1-W{B7tmlD!#WLa3^5Rr6h^s3?-Som{-XV@`c`$2vU&G3digbcIWIS zE4}DL#apt{#1uq=Fwy9*X2PIaWl|HeHsFqpp1^m12g}c2&L*E9@5{p|q|W=!81zrq z4w*LX71q$Oz*=J}B9TpVv_^t$Nytt;R%{q-!8#W>8tnXpX;X{A+z%fdM{=axmK%z@ z8P~m2huB2i;LQXy;^>Lo#Ll^fjk|8Z3#Wu>TfNSqVU};=aq0mC8oHGqW)PHmSJL1H z@`D4oq0Dox;=1`^Pk?aO8q}^JM$+nTy-k^;l5BC|a(*9!8ISIeJ0LG|m@Qmnmu{wU zV{%yKG0wEE1CH0lVYLaftN=XAXN@Twi_Iqys&CVf{IulqL2t4GkHvAhNqn~(qV}Xk z_)Jh&_lVChE6Sth1mG!b;qPU<)n9{e=kIR|w6^i~?u?vI%rEl$$7>!J z6J_~4ht4}HuK;orK)=@_J8Nw`m)vDZ>%oRQ* zaX3MOv%b`SEMd6HDpbWlue*^3`CQyp7Qqs^@vy|Z{G%?pzT{byHPU&63YSx|HEx1+ zpu@;QMDvkW1R<+u=!8i%1O`JoJ1tEyJ3HMg8xy?@2vjC?s{TVVMw>tk*JM zU(LHnnkBQ`Nf1nzwGWx(4IDS+lt+agnJ{+#b02keJs=&p-MG`MNXUp5_>#+>(wHV=j zd=NwtfJ{Tf7!shU*U7PqiUNe2E--_Y-iA9d4<&f{<=`VN-@H1KdOHB)AumlFY+ATl z@qtIWPoObH2CYf9oE-cLxsq>6p-;I=?S&e3j)F8ro~{Y~H{*vGW{8GIZrH$9y9Bbv zQfk>Ia)ut1OJU`%(-X-gp~4bJXWTQTzHzJ-8=I}OU7yU5Jn;J)u!E^V^%8Qm;lX8W zS&EjEg+ym7=_OfGfDQp4z^0e^P|GTTjA{Ku?ligqH=-b1*QWJ8&C_9;>b40tm?;<+ zi)sYc&+1>eJnX+P{U&h4pKlP@01l4GXS@2(M|Q!%B1kIiHf0~unwDzfjQ-pg z{sFubb9{6nb&OBW)h}ZEIE)Og@tDmk-aV{hCDtmW#)l*ZYm>0r8Ni8z#%&G)GK2oj z#&{me*@gWnmh{IdaKzW!>~}@{QX-loxePMK2xsDV!VDEt#n8hp<}wE8xAIl702ks% z=3SH1x6;B3m(zfNmV-(k~!9Mwa7kr1J$T9Tq*A*Qqe|muRa`ZYq7<0?OmljQ9`7M`;P<# z&~64zwgU`105GC!!q%*8y|mF;$TogaSL6mA|12^E!>V-@rUM$5%QkE-E@(%R2Hdp)#p0+q=EgHgeTJX?ag(kYEGKvn<&Bd~ipp*}*funL>Ky9O!=Ss8+_(y}dI}=uvQ4|st1eV0*5MCqqQ(*0l141$t~#VU zHh^-1>5#y%DCOG8s5*uMrnim>02*#GGyYnmIkEJRyt_3>-bHDM)gnE4rm~*M4IPPr zOoC0&xDhdZahdR0qFXSboN%2-1g|>n-|9l@6j^=lQ~l<$;~C3roW+{Y*EHAh3WGDxY_==W&@!hPnK7A=;n@v}r#;S~ z^{vWiERJDlCkk|Ici53XLD_7B9m1W&XwCJSoEqKrPuyx;c7zA;%q@<@xVQ1)4)YGv z^>*70&2=6{x`GaxC6a3c6G9WA8n=bOV^r3cYWO*T2y#qo6LKcg8^Lxhp1eY9C@-oo z9*~EKHjW7E^56Vc2&Yp+`co?@USOG>L171QG*CwCNDY(SV5S*GCZ|SF_cxSH@REjr zXOMM=_d!!8E~{=LrrmMgaR++{Qymhx*MONOwp;E-M>T-A@)fWEFOtgd7SwY=US|%` zz&8B?<(!3ga^ja)vkzZt^!&vUXg+w(G`!HE8HqBNH9RW&rzrheF-k*iaGQg`^KqLK zcw4tgWqVQErzl7qxdFwYuGjiXc!*Kv+`1dCUe~8^)^F0+vlFtSVDT|kZ7u*E1=H&h-o!-!7{1Y1rRuSY$WrEEEEA4k&>-qCA8;S!F*}gA<5$H3^P2O7Tw!~J zy5{OH=Xhrdk(UeeQLC@N+;8s0gY&m#2Yz2maEh)gpZtA>cUCreFa86VkRAz)(UlY6VaH!2_C{ zj?#|=MtLRp%7G<$G-8d=c+{>Lj082|uq*N;6Q6xu24hkMu>jkl&kV;F0v$)7vw)Y3 zmd7^>M>AYprp9bBq7kBEQ!uNxGpVtn_bSzbBGV%W^pn@fW8o-x?J?Tyv-@smA9CL3 zGh`i2lQ7w2xn5CY-k1=;gkZqhm5x!aTtd?SN^9$e81zb}cqir4^|?;()c;&qUm7rz zaMLqHY%;{F!O0qLh{+~`$u-CCamMy}LFawFfA(HpNXKh*>>ol*4`q+)$m| zQ@8$YWS$w*Qy;W#f5|_56czM44d5Z#_@YoB*iap7nv9#-NuoMxrv$iqxB~bPs-7-Z z%he75twf*uoBE3fqNa;ctQx)L{zx&NzmZofvw^8Maat( zo$KsC6hTR^UMKybp@}F72a9|qup#Nb?>E;qI!A-KNC7owH#>NTIvrKMv#>0T5bLT{(%w>P)1odPM10WQE|*ROtQHVb^r~`qSb^%$zd$Mxyddjc z=X8meLgM~bs*e(Cqa%8&kfxE2T~W-E*a2}Kk_w$%TQUw7dyOXQ_0esLqgh!e!nK!3 zFI5L~aI;3)h6<@qf?y9iZts4HAwlh~lL-^}ogR9LO#a{J}T=SxjBvxeK1LVmV)+6uXS~|MSAul1c z*J2Qnl?wxYE_`eapOMHSPNjt<=;(|RQ7%{jZJ-|doTxBBPL5Hi2g;~FNdVg91sNSE zy;gfN-iF5lF$k0HT@=Hw#B})qz21TDvnb6rcf_dM`OudZ>I`dSLpt(Fc-#>UR!dQ0zb;q4y4XjvWZsyj2HKO~9=jBRqEupccp-!8ABR zCv0@Bhg;qoxZ4`Uk6!78CX`#LVu(p-kSOmxaf6O0HAJc_I%K|8K#XK zrL_(M-dp*8wm{D0h7;$QL(Q4_PT#YbLvXwK4HwC;JM>m^m%De-7ao)xspZPYDf#gS zWwiFNx#8T7(^ZTuw8jv9U!Cl@eQKap2@~37ZCJa0h{cKVqSOsgAwA#|sSSG|>ljeW zY9*ec5=cj<-Si*_iB_QXmn2UH5axFQU<>eGUp=Rt8-9!7XoQm8B6vseNJW3A!86e4 z#=80eKsuvL)~Oj1?(oNN__9U?_zi)efvHV^~ZmI|RNE&XJw z;i6<`Ik@U+nw6Hp$VK*DjhF}ck<$~FA&$T!su@HK5ty2j%kWU!+Yx4*hQ4jvr9m*4 zz5<_AqxXW1Ceh8hIZX=QRUyb*`N~*;UkkJOg}E1i=LB=ammxS_g`@DjJpPGp=#LA| zfwa8^IM4{|uf^^2MaV}?2-g>>fSx{81UYuK(FhkL*|wn}M}Ky7J)#=mB{#WbLlbnE zqi{_)NRQ71gMM7e_z(pADhU=qXfQ+R15fZ>P@RfK6R8$K8~6JnF{mqWeZ~aW-tx># z&Fejg&?f*U6N9Z(0+(k-M0EzS8?F9e;7YVaS|>3|MpwVS>PIl6jv~7FA0~|FO)rfK z(W?_fj##@P6S0AbQ!UU@W_dz zX=wU<9b$nKVKanL1lkZY`)27u$~1d8yuK~HjqhUGQZ>OGUkJQ9JeS+xk;P?<@x3{Z z!}I_=niG~B0PwAPq)4?%=7{66Rr6^?sL^>=%b6|vZYeg32sJ{V3qNk&K!SNnN)VHb zC()0je!)|lB%1(n72QzQgKVB3+4&3240q^GK}N}$W@r2m&${};=v4u@{8^h?1~$^6F}sQ#cAd$bB#I70yO97#l!eT$>aS` z{ATWw2uI0B8X`F-?BcsXRD9@O63T;6txr&h;TsKhUiP)VjvvA3djfl8w=jGZ({RCD z^!FnpduEJF#uw|z5~I*NyUo>OwT}^iHM;OWW&(J5Kt_pZ0ttVrV&h3Ubck8UK3@>@ zDHoX*`H$%FgnoVBSk%x4_KD!zFjUOPzAY z>zhT43St~n=?I2V$!?i1B`(vxvqxaPw$a2Afq-E23udMQnTKs7U;BpHv3aaNFy5nV z0*0nD!Fa%1`KnofqwE{)ys#L^wJwmIR;yp9@i}spWs(|2!IfO6*ZlV?)pdoqnnP5I zErF98YKs#YMuJngG(tJ+dJHv)NCoRk=VWNI%TBIazj!!aHaXe zI)6sbMp7M}Os`}*>E91)x@?w6gEX~v8* zKC-dh>NdzPaQNQ(4ov!_5o4`36J(hANtQFu8`>I~sbu%w=8h$dZcb)oj-QoVIV@@1 zWfPSpBN-^5(5i3oGdVfRjgCtUHy_~UrURXC)??aYag=T%xlr^bjEMo$ZPy0(q~Ogt zIj2V2ZPHCMiHdkCqO2`?fW`sys=M#I^F$7`2&gx(xqN}cfzr7y*fve6KamXS4r^Q* zq9jX9mkSpp^ks*7qr+CiqKxX|jYm^xy zN140CBpUNXRSF=}MNIqwr`zR;{pf1}<9o(gUqUh(>M;wqj;ML`lQN!E_0zD;mCU*X zfs>%etpTX8InTr|ug#V8S48`z#3}Z6_;M_ewSVa!eE56*hd=)>{d52Hzx*%!GZ}t~ z@ylWMkE5DDNE$ck^C|wxCyS~@xIoTY^=z^V}H@sky$zAJkOCEdlxLcph zbkQ>Zjh>&|m)$=8#81}bidzjMVf-u zt;8DlhCb>?c68J*?0?G$JKU0|(?S~Pwi566SV-TH>$XQiQej$%9#soc|E;9o^NnJ1 zCX+nsFEvD=5UN_w!d!G~kR;gvgHermKQO$mS<^T=!-FCp#gW3)N`$^BEJ|6d9(xzY zofuDw9qmmB)qGIIr1c){V?^xfLnq?XV)Zq$B-x4Sd=&;PUkRB`w9*Uf|PfIOVuk55?8V!eebCz_8Gwn?zBh~z6E7aZp-%RIb{ zzseAyaQMZ}5=epa&aGTs8!Rce)3*ZOjp1pSM@B8rox5`GJlhxEiu}lncDXzfi!w5W z4&fX({@10B|g3EE>XvwAQn8cUCbYG6bHF+C~OR9#c;pG zN~Mh0FW0Tg)GW-l71f*SnQT+B8MobR+}LcGas08*^bC@@!S*)UTqLFzj0KB}?%V7JWK(`p<^KO z0?2@fd!?u#@xVYe*uP+_i#9+pWUWMLC_$?v#z{gSk~~zV}^#4urx)F6wT>oSJn7JWD_^Ds zzV)r&@|%C)TOWS#Z~O;;=8yi_pZuj?{+06Jm&xxRp)D42kCvQfyvyDer@34$=_I$9 zG>xm~EAuhy=m93JGSCG$2U-O}ukovrTO!!S3wIQhPeA3n_m9H{S=1{H_n`;p=Go`} zNHs!GBMW$*J>~{gmaIkzjGnE_og?D;)u|^F-f)g@Y*c#qjZ|BJ3)M2SE1ui6FYvIW ziPRE_2-;#}qP!SML#WRH43-#vqSR*?n9e+>e#B67LMbANlck6=;2F;6Euwm+zG|e9 z?XmuuxX-Mv1U7|&w|&AaJbe>s?Abr^6Ib$S=e5L}cD2x#V1k!Zs$CDkk)idA8LlBD ziho~0uPV(}vbrtiVM4|jDuLid#zQP80Qs&wKj0cv1eb>RMeJB3x|@ZPS9b4b*}|D9 z6J%spdpmrmE%41>{?&i?cmA0_`4@lsAAa*!HYwjB0h<5Cl|iVOh_vE^lz74elH-Ybx}sB0 z3<6<`1fUHBKTiL|I+0lkc>iTZ<7G^Z>Ziik4$5J6whIC=fTFlgLnu9b#_DvM1`Z7p zSA{0Fqh&6?!%i|7kr-V9OAd0|)3N1552nkJDMa(iq`F5@9{(7tc$nD|+?raKUNSj3 z4Glel){7aPc(Bl1#te72S5jF5B^pyUVS}LyDx57r%9;}x*5@v-g!{P>zZ3-8aEWtn z_BubVW@@t2Y|&w3K-W94Tff({3;d_nX`dCMJ)iArlL50O@_HkTBO9Nme(J9$&KPah z<9oa_x}B}2{rZvOqpNk^?|a@AxoaD9%>%jOXG`GbfPyCoh!MT^16xleIyYe^U&jmW z1SHBsKUx?iP_LD#wmyhZ8x7BnXstQsoHM+aUj#LUAab@Vn+{@TL!XbHbP^Eg8M9+l zVd>$onOvKGcbe5|g=Fs5Jp!LHYA~m_H#!!BNydUoM@k~8cKYfv%e6MP^H@I8ENmWm zJB|Jkz-d&H^DwkRdlE#G9EiDor@;W<4qui9Zq7gW#s}~J6TjuR|KN}O-aq)`f9vP} z9`k=k%Hx&h|2{wMG!HlbbAoyF#K}DuOM?8Jd|eSML35jnwrydTt`}%vM~mC`CnY_8 zJx}X5M}xq3k-ZoS)a|$Eaz$H$6XCU5)qi{^OQCRLSc6IP^hOp1`6)zCHkYqTFVQUe zTG-q!17DJeegh=i?k9CjK!f#m{jjo!#4-p?OAGE1^f$m`G%AsvDR7Pq;l;WSjKu6- zsS{U_+4$C6UsFX(~rgAaFlii8k zw%i#`f=wDaMAY{ZN9(P8cP#K5f8!7Qy-$9_Z~xst^85bCU;bM^|M$P6LhOR!{Aa$) zE%@q6u5{-dYrfYIk7>Ac04j6?Te_|A6&PiSh0x<=*9m;LREBc2FD!&rghdZ89k1t$ zwNM4aLIX_0mH97%5@Ec6Vcz+2i@?!!BQk=Z)j2Lv@)wAxZ&%S6suTgSj;I}gdmk{m z33B7eCc+{QVr3kbK`;P!uR&o=#C0CCMs;GQGHD+=(e9Q?!jN%b;VnCzJ`Zc?4~fBL z-JofBuab1?5UDvVkvJUUGg>{BJYfTy-;s@TbDf=U%hG zluxNL(=u5$%MjVa%KK_=bQXdWKpa4BKJ?TV@WKGiO;5vhD(1Ce7FJo$HRpMwxqe=G z0Dy>UBIy7CKmbWZK~x6W!3M!(Iy5>Qq~0MfBSbk(ZGm2>bB*nm>bW%-{W(r!8%A`y zlTf35DDM=D-C6IQRFO$<0JG0|@N7nIrbBtOcq(q|H=c)y>s@6-Awf{fzHAM-zxwOT!H*( zw;X^&1WMf-f5;s-HpF=>4JXz89j&+Hmu~@^yl;Kn?{@uxKk-xl#Si`HkN@?bb0<$) z{&LGPxR_WJ&#z<`%b_6n)WIyqwR;1CqgEWO&GW3*y>g_v`E6wnR{tShyW`7QxB8qQ zK*rYU)t_b^Vl5`S26z`3=dl~fPPc&`GI3@#x#;9QRkp0ih@zSP7uH<$w~G1!xwuW|s*C1BVV40rA1o2A4jz!zLKLI`;0_EZ`Fi9N;U&B?s`c6@sf>vtv}I%h3zZeFj*< z&t58+NKUDfjy?UYMEa6R9S^sip_r1}u9ELvxfxm(puj^40HFx@=rx>^erlNu(nR}U z9vMb?GU;G3L_b6KAw5Yqv@Ze;&CtE3=GodcEjo&fPU3N0kC{>+=$_}{s#CH(#9_Qh z8~P&UvF#T@{c><9nTpw9hv~ckn-ymY_QHIwW5Q<5`G*NUJE=5@{kf3$knsy`=G#Q0 zP%m1CbsqrE(wi;l{9(r5lmibzKp2C+vT4ePNAICHLG(moQSs^E3vJx_aixWgV~u_> zbAs(Tw+4@k3h$UGtJ@d>C6n7GZOw&r49B z&hK2eLlz#jntYQ@kM;rhSau{Y4<_6th|c>P?!I(-jE}$wPr8=#2xXq-D?AB8uj(d- zYLweu0pjewW~ZZv`F;5H(7#*CdJVyStbkA*&l78-u z5fr8aN86SfAC!i5NplQ)HUmphPRV5fbX`QAD3INz^~fBOyG@q`cK-s&hL_NcVuwl8 zB5-Q3b3;>W5D_|o3F!jNLmV)Bot=A?Bc<-9j zn5Z@1k!+eSzXm>D=HkL50l{;W=Vid&VF8=>m~_on01(a@bSll_o)q z{M}?pubIGfV1xpeV>+%A9=Gv#dMn=r3$*KPJ|(VC-uvMn`}zO)_x+{+?XUc-r$k@g z_2tU6D|UXf(&=t~%PR?xXme0Yc7gXSt_zSythx6-im8Jt!B6rO;A_CXE+00merhuQI9+vbiww z$aEA<&wYqdJLtW~WYkBSM&p!V3EU3vHhE$YJ;H~Qw3T)sFVn1F9~M#GK^r;xK(J$( zg2qR!Q;^j=q~qB$*;`IRg1Ko0VZ{Jj&%>hD3-S76a+Jw>QtX~B>YL9KTlzubun!Kz z>C(VsgzJNhr{#?PW@_>{_lg=H+tqJmYwps{Y7hi7&YnX>nbatM!Ez@0g2-PI+D$68 zXg1q-%3Q)|R-Q>_b0Fspw+vi8n+?{i^v;@r)q~!s&WLshk+aUudu#fN#hd_S?U|># zS8%M3RK??l$kuJ<2Af1k_O2(Y=XMXy<08Xu46QhElFhL%V1=Qpm(CZYNk&rl`o@6c zXOPPR>z*)UUGsb!d=4`=nN){gGV6|^u&7->!XRIx=l9ovtuTvDfDi0C!(twSOu*nO zAKgQ!bAB2HaCt9=B&^^`m1+UN4jPA!!EC_bjZ**;RY8LStnU6MVg&|HYM}|9DsJgM z$+DfyB!Q@iE4e0s^{|arWib| z11yLgCyE598Ror$R!~U1PIX^GkgUh0J~t%}vveF>b&#h`2*iP}|G9Rm)uZJ2dS*+1 zf@nPRq64Ebzy4uwx3qQ>*PW5d_-uW~5*Jt}^ynvDC29ANx<>DCCDmn7578YWAv2B( z;0-4QOG0@^c^0lWZ2PDap8SD&kD0?;`7T6alM_nMP)jHRjC^VQ>)TzMj=t(L#MB5}~XXHg(%ERx~eyW=Rs3hL#r>w^WI zH#?4W1azFH@<8uQVUR!f9;JuDVa}J)AH}5sX=qd)jvKrrv8vJ3DO@g^({!Twa;0b{0#%VFO(bS_pV01?BjDX7%uk-7}I-(ReI$E6= zc0mT;HPs>FjtP#p(Xmj z8o9oX#|Qa!Cm4EJw4=>vS8EY2g$ z+mjvNFHS%DxnKI|cmCHu_B(#?&mI2*`?kvxZ=NkM_!n1Eiqgkr=4ja7S2%*|TNxop zD#Et1&ke#YD27ye$WQ&0`F8Uo7@nxt`|StP|Ir&#THH{R=m5GF%nZ=#Ms zo$Nj1%OJsqXly^QNcn5kKTg1X4QXbiNQExlWa>A3B5+zEdV>gh&ub##$AuCz_j3Z5 zbR&b*Tlwx-;5D~RpLL|f_0zxbEC1eq{uln6Km6mb4{yI5!MK&*$jNQyGdxa`iQ7V6 zz0GBAv)yp6mZ<$^Ag||Z!b&#bMK6bo=DF1c$Lfb6h{kB>@shsks{3*R&(zU1Mn5RH zPp#c2);9`B;|O}RPJlUm-EAJW_d|3r;FiPt~Bx*rNax8EUW#7Rx-lX%%=mlM&rIVP{Z7nB9hXNU`c`KP(Wmz|R^J3JlY+ zPO=DFe>NIU4~bQ_KWtp)Gr)F;oYdH6ThyyPCPeWBkKWB)sd|FCz#A8|Ne(YbI;Bfm zT+fd6P2k5|CSt}|H#1rd+m~(UJb8?mxz3LkvsjWP;?py8e<)>r^3gK(EEmWZDc@s1 zqrB~J`b^a(sB?Z_+Y+VlO-!jr@oRyhfx`8IwnlNXnqNX z;Sf#_P;ccsX@TX(XX4`9PxSuk2Veg~Km4;l^MC!pfAe4eSN_T0^aDALF#NWb$Tiow z%-#6PwI284q>+B`!+ZkiW&*#p0+()n2 zI!S`e67PRBPYmva=o6uw*IF(J;>Jrm-~c2plZ1o@w)Em6aKP1D6_Ekdi&|JEx&bs; zq{67%qP5;2l#?63b!WfGEHQkt zl`?wc3#=<2jVe>KMg=Q+h2P*DWsb=)Oy5a7u?U2zzkbn35}c+_|D?%mptg#TzL_y~ z%xq%^m*A6rcB#=qQz{+T7b$$+%6HxZ9FfbIAx_us;0J!-Kl!1*@{7Ot5B{}(^*_HGIIcA~fu8ron<)0%sq3RTFf+&ETc4?+eQ9g`54{vl#0-INpIO3Q0 z5{L5#(L!G(B9Ch7wLlxCgYo4*%im=Q?1tOe*r0f>d^cMN9L*2? z)_h1N_CoE`1ioB4p_4cmF<%o$yp}O0%OQ58po^zOCu4)D7{~LP_GTIAHnSny!a)Z) z!FOH_9xGy6-SAE`NvugCuZ}>xrb}Gya*j&y`(>1K1PUhh(Ww5Zxf;ovDK?8@Y=@l8 zaW~gQOp+iFnY63Kaz?X-HW#D*;tue0mb2>TMEdf;Q<{hOLt%8=bmov*1IxOmz9Htx zR$1%17zc*v*x>mwG@x9b3yltcHBU!0oWX!84&rR8oA()4qa$zNYKz?lG1^k6SaZhUgXUY8A58t%5H#=-n3lm92Z{C zIOunFq)Tb*p7S$=XVl9On2c;jC~F1N7|Hs zec!I0L-VYB(PuAP&gN@%rXeSc*HlG+8c&J2VuRTM5iwS+KoKiN*wT^@5Dq%hD0wfk z>z+eTy7h{P<2Z5=$*SV^TpS1Zga{yZ1WW8%Wy=`3#KPrdUXH7CNRm%6hAEM-nOs4q zyG$+HJp|FH&?{QSXiMR`M!<6>lmvXlYa&J@G8E^Dy5)2I*x`0mCn4B+CWCq0;FEl- z1CII)Ys-lDhwYJ=h*PJIJzP9p8icx234&nW_15}G_5J?Ql)mgxf;#vfTt4R+-`z;v7Y93?ujrmNj}j!Iddk*CnCbep0{9Wupq{6-Msf3!o55qI{0hZL%Yg_q*3*qz(y?+$yva+ z1HS)(wC?n1j3lYF%plIToxpXKcSI3{b!y#agjCorL)8$H<9w%6ob)qT*y{mQhzUv1 z`}m?BKt%0!#*M&>-}K-4Va*hi|6OOzAp=JL^|R?&mJaV$UgnTkmRgc#OPeUBTEB<|Zs0 z0+x(1{!F)G#)d+e_xv7o1kwM3P>MJ=H!#Y#mDrho2AUJgD<_vX*Tpb za-;CknOs4y0myB~8Yp;&{#+_XZ178eLb1da5zbHinaCG^LwwwY_XJG;ACbNOBK7 z_)8v1erC%6#}XWfxAGmZ!1);GdNIrJ?gGyHKK}6I_kZA9Kl;;pZ~*u>aYLO zFaK&#pStA><=1rTnKn<9wQrf(L>VPQo($M8n>^4H$_VNMCVCkCG}+2X;#g91 zA7OHoZYJN!`UsG`sSp5~<9ZZ2rZNLtj$d|Jq!Y@vfrbsZXZEOhQX`Os$aeQeN1iUP zUBafT9ML%-*D6+?YPY+ZEF%HQ&}HC#bh9-c%Sr2DyxmZnjPtkCq+=lAqn+(~o6_0y zL1lq#(lBE;L};CF<-2QvrsD(5B`Ne{TF}cAAIw} z#s~84kvB$#SsdMm6gHFNLiLG5(bixvqI`}-xW|#_mK#K19>txkc%P0JDC zrAM}BX*pt}3;#q2H|K1fAD1!J3|xt2)iVP>KSD@6Oe5!s&7dxeys+Z&;7&F{(9qcF z(2Q-|w|~5jI+p=jNCG|)vQtWxxZDrIf-TB3-COC-^|*a{NavW6EhrVS8PU@!f>@e@ z8)qYN-1y)C5BK@@WZ@y&65!Q@yaEI}n7}dbs zMD|F1>I~SjwQ;0oH(Wf+hw0MHqx-<8=dg2-7eQ?*uMPW3R8)u@fNOW|mwm?X0s4HW ze{^m>ANH33JyUK@9^zD`_jEqf@|Rm(>vvys5N9kA%b}ct-c%GuM}+yV`%D(pyi7MQ zJI{HZSg6l@Z}pq2xI_VTC}bSD4y{;?REcl3E#ox_aZo|vJeU+2TV6gFd1g;1f+5|+ zbL63E!g-3dD%5&%y`s*3JMqQemSo+P$hpIc$4I}NSaW?>36+@dI47}E8-bmutQ@J1 z&s>bas2PGps@)pJaa0y>&xS@uKxxEhO6Hn`4}I2q4c8To2Ye#j*t$%}(FejGQa?Mr zVhHGbpyBGg%IL~H{J~WALNDc>F+iV!Mx<74lAx($wF7@Ue(4rCx60<{%EwPc2<8AacmQ{>Cl;OYX(zhGodZe_ zR=jcV_=4s^asuB*#Rm@ayl7rm47~VVf9JOWZucOV8)~%;25~nS=9lAe1g{^*4GFql zezo+vbPMKp3mZ0zp9UhgL~&8HSgFUCSxRT3>;MUb%*D|J*U~$cX+MAgsS&w1y%$o^ zNtq)99}Jvj87(q*a-a3&3+iDXvFNO!Gb~ggRl(^cn11z68*E47vEDTgtM0SsQ0mit zGZFUy6XPh?Y?|*jknTylvB=NFW03&P({ZSYDRYm=;b@o@2FXF+jSgQgjd<=k88KcO z%!X}gBH#V{cWVOvvApYfcpUP)ZjOfXTlwx<;8QQ;QaQe{_YMCXQIS^318kg zF~{Os9g7GeEepZM-XY4R*aE2*jqp{%7QnAMK9XJJc#fBvmT`<8+WfSVRA9 z)2nBjO%BkA#Hd9Qq+L-@#4CdJt4(Y0|a z3qFGZeBuryf+t<@z@V_-szY;y^K6fd?$u$$P&F?VC_7A{o3iyCNuyt^JoLX9^0ys} zO4jx*+Nq!MSDkAx)!CB;=g;NyGP>pNEHocz)Oe(VuGBzqAT$0Z^k(EUmSyueg5bBo zKV{%+*87eug5!IXw{4lPZ~#2-Wq#ssD%UeVIKeu`Zd{G+-E^qI$Eo=r+FC78d<%sw zg$?h6i`yC{_jX93HVTUvBLdqWe`Sw{Om`hUi9zToGqrI(El>Uvqa&NFfkKu3D)Gj( z_Z;cimCF#N3R*XeLd~gmgf@IY58>c9QA;|H;wLo3YGnc3sDHst{!pZ_^lFXA#BGP< zx!~J=VT`9*WRjHV5rKu=qcQI4dtKffGptWSd||zn@2CZM{xdDj%`*Y#P2A&apZuL) z{?*_A$A0p6{rqM^-RB+jJiqa7E$Z+%bf@q78GeMvlIp1U=vP7lh~yP(qY%VvSp zj(1TqDdK!0128fvdfuu1hboObj(O~_Yci4Cx;%wnz+#b*aHa&@7b`;%7>zoUA~1)s z1{X08iOG8lK{EhoOg4VpPG|Ju#rmtlkJK;%zJZ30SCC#6V+j$LNp{r9FvxWu6fsO| z1B8C0z5B~YCS$G_Z4^ww4r3MO-2sbmquG2|z~cSrUcwyf^_%9Ja`EKuw2>AK`uS zC9u8$aAen=sJ9GF(TTb#gp!x1Z^Hcg-8~}TL#65ICnIRstY_uDSa61!x$Ru5)ED5s ztU>-wIJelM%TvJQ5caR;=uz@qB)B>TzG&8*hp}}F&M2A;Xk%TU&ndF^3BkvEyW<6S zWaDEH?-0k)7t4kd?u{ZyoCoEB!bSg%9Rysji5>rQ3=sSz>$GP1OVA+aH)DNvndTfs zVXLXJV>s}!YrN_#ktz?^1;Cg?518~vX`A~A3EkLjxMBeyqNdHsEHDq_kXZ7eQLR#* zW;8QrPl^v~dMC~48)B~pGy~rE2JqeW$^Fm{(IeG9qJh&y&+dn>(!d!h#cbK46a^rZ zi|w9VBhaagOoJX)p^dwp4bJXra3@_S8T-Tz;;np%7T`lSWo^M1^|fA3!Z{jkzU(GH z`PMJ}>PLU*&;HHd`tSeoANk*ZwlVjre|;tD;$p9OiIf~CFLV*j!Fj;}O_p670F*!k zM}EiK_T9O;4gV5oJ^H(E$4mhp>G37X%;TIw>)2P-p;WQe4#p2$4k>?}Ujs`Fz~PQW z7+`i_m{+~15S9pFyj z12f2TK3E0`iooN64;gYUym%MMV={{#kpYE(S87y~`SvVY2oXYnno$jVWdc6+x6^mQ z0_VV{=iQ4G=ktQ#^PIZ zo?Z=WR9?mmVyctXYKWHf4s#{g`k$fX>v9l?`D&=67yY5~QdruzavhP+dyL)pQn7=i zp4rKYGsl8)R$@8MR*wx$j{-G6VA%BRVKK=j+iiD#tjUs%FOsXZ9}?46oJ~+f8TGOC zn5+w>UfDl-{L0-x0Wr#U0b4#y&wYCyxLNWSX_yhUAPE(xoMW8Ou|Q4HaPT(~TjEBr z#^Q-~>C}ZCW-zoH{eUai*K1pO+DxG&aMP6lL?%Oo*=Kb$_|y131D`e?F)jyY=A%&W zLty)uba|;6^{0MXh!$dE9ow@6O?{0|_IgZ1){JtOi>9feNflPc?}y9!Oqvp7Y$iVP ztRUg(XLGbVTVvFdguVKrY)G$WW@w1138W12ku7RM)onO{cmW4YJR$j6}S``;i+5nW>07j@@^i~E9;3^{+VoNHn9xM>b%3C;i8sPGU~f96d=aDJGW0hQ@ZVi=Yy$C`}Qhv%yW> zd($?sdz(JbGv$20zkAJK-p~8`iG1GV@P}(=)^nGuU)O!z_qx}5)@U>*QEx_-wlF|J z)*|Wz-y$Nt7C;5d9O<;my$|bJ*f$VYMgS^fmae?GjAl*DFHg+z>$)aBgCO+yHSt>T zaJ-Wi=)+XJV%0Y6tcLt{C~X`fXkbL8t6<R+iI};HqYf7AP{a$TI6W#m zB?S)mvM$mBVsjW2MNNnxaWRAdkb(r1;vKZmoWuoRAm$4*2#DR80EsKM$sjMGSdq`z z3L3vT{;yhq$s?o9pw>Gcob@Un@%;4E^i=EQy&J~<$NR6o;f_ZIC~yJ(wN6Hpp$ans zp~R1j2x%4xPUY!U0#ZR%FJ){Xhyll8oce`zg9`!_5Zy0}mgiB4H*!D;`4D5|8+hgr ziuM}R5^(_elc@AU+jjl{kS1aZ;3UkKTgdGuC1uZ_CIZIzw>0dC96byPYvmGF6w3Q5}DBq+c+ zq=R}H+eIvzQ9*ZB08uB|3QSSLTb@GMlt0ciSi_7FjG_Yh>!O!7+L1@g$EXf?`e+F& zPlD11h*J{U_zO5T<1j1}A--q?VBrmtoY`P=6<4$gq9|0QDHVw;QCdniD-tWCA}L#_ zN|0ia69hs@S{&dO2nBok$206T%27-UsT=l5QG=v~s+Z4r3l~OvTodheHSM-Knd)AV zyx{2fd-remElCA+QuFbQsBT_ za`jQ_#u>t}_)w&x24EUp85B44gDN=+^b*<*hImFhM|4DX)uQ+w5;Hsa)mshRb)CLcz<|XsEKRSj2!2MD!rI>g-r54d!kY zkNV1zVU}nho2f%ZO>v>3lqwTpK{c<%QV>T3K?(pLzd8Q1Er4%yk@3M1pX4)s)0>sM zTBrD0KP&87=RbdS<+=?!U;g5Qc~NP670h4%#C&L=F(CL&Nb7iHF!Tu;r9n0XnR1j; zQdDL?qt|7puL))$E_j(6v(uoELepv(8OWkB7^fLW40MWpmmD9d-~DY)!W6E!tqWo3i#QY>Qm zHZagLh_h0I3YD-2q8cayX8<7^3SmRXl&Ue#+a2!JE2I(7y%@6r+Q9@6w=qgKfRVV# z4UAn=GJsDVVH`_3rgH_PBOoOPZ1IOi!3=v1y4yL5OBUrn*A5?ETE&T%}*}vXwj1;{=MVE z1rVfxpjlT~FkhD_0t<^-{mt=z*#Zn2g1bbT^BsrRv?!x@Ck<8_ zjUy6S*;H1UHr9(7uiRL6tDeyUpo*kK%o*J*h0U2BcFWF39+DPiC0p8L0iQXrh($XP z$w-cZ$e|!bvUFAA(gY1;2mvX=2kz`%FrZXR70aVFNO-*xc)6}ljej>8?Wl81bz;48Xf(Vfo=z~Zrzthe*l7ej2ty++(@WuEw z4`c6{brffKR?9e%#Ey(Hhz&HOFk7SQoz_IJrqDLzh@eI-Awr8~0cK-l7;74wW<%Kk zI(%FTkWe(RnoDv z18+O>u^CEZ)Jh9C>UDMXD=UuVH4>H zQpzRv@UBpyBZLL1=a+((0YQ*l{ZyTP{O0(tw}6f)k*-H8Ke=Y`*GYzc{Ee5bPw`Km zW=5vlzVMv~zV`ilH*Fn#y5EO!Kyf%CjCK#WcpXmN_;OZDq!?c%Dvc|r(9$dxWe(9x zAc`hQA56-EJH3g(68NIbv8GR0&`n2@%9tv~0KMQE!QlY!fbl6jUm8)WbaDaUKItNyms8f$R6ZdUP@Bk9%0HKriZpV} z0V3;S4+M~=cCfSfraY4z9u-D$VI~5l0!0fXEbw8B9|DmEIuK`K04&xhnGsK>+7z9mUyC3-vz(spzDWR8S|#g_EW&4x!JU#I$w_ zSk6{}!n$#R5s(SbF_(x4EyFXE^iIW~cr0-hZXt(<;v|{Gvk3x-TIa)%;J2cIRCyf4 zAYdIpPehLpDTZljQ%=_R%3k8if?B|4vA{`(d;+8I<;F2}qE+S|MVfYMcrp0ZfDhOQ{k zGMr^o9TMkP_Z1?Jr7%<|4jD+e;Aw#BpuG&>@O3Yts2)$rDnb{@MMjl)RCHQ321m+3 zN3-OaWMuLSdEPtMzd8P^ED(2O-Ky`3N30^4;Q&%%+YfZCM5(CLlhG07?PAvA-A6urg*o_fZ`iMqiODE%r!3lP zW4Hj6C#7r@pz0>?IubH0i(FLV0rxxk^Q8U?r!}NYjkAEP2Ex5k*?(6jwNL=Vk*nEs ztVE%p7}H6W_Og$wFoIQV*|nf0N9dF^YVMILAF3tQ`Dj~nJW3x3D=TV zQB=o=gl3r+EJX#Wq_-!WG64lH(@z!z`nD(pnr^LjfoVftdbzAoUp!JU<7C*`>MyPA zyhNqKP)mkm(QbxYN4O@~l6gwH3g|Z<|AiJH8sV1Mqu@9-{;E>t&*E5x(phFUu(%Dk zb8`B-KYi>A-@be8W7}9z6jFlyM-SNg1Dixa;;O`qoGVHAq{BE7$gnA#0zpfJ^fQSd zRVxk(Hx%r)?qMRLobhd=>V|Vv4x~-Ml!(nB61fntQOE*9vss-6>W&(*geYnRR-rXk zk-rdf?1n_x%Z+MMK_Sh#u!+V>T%@D1bjUFjC?HaprYGFNlEYI%Af5DzkbsD#4m60s zM<6(pcd93Wqd4^x{x>MJoBv$eq3+nF#rPfL>@ODx&yX!_iFg*@)>PtdT#2C;9 zfNKmP@w#-y)SCE*aM4MCfrU%mt1Nwi5yw!8)irqrEqtKNNjXZFgcHSw{3q zr9DbfqDHf_sRa!Jfi7qjNc4ob!C()nNhS9v!xu@SW7r518c>EdjDjvzda@OX8Rxka zvw;+@1<@MzSI!W|p0MD-^l0RiTU5<>(<9K+5E*Q8!Fsq*c|!u*AW|B%4FFopE?O}= zA2`L9G!}oEeK5EQGtDT%&P)hGN*~eg9T%md9e3HR(8{6>3YXvr38qBrr;#BJ_$9*$T2mbW34VaTP!w;_z3lMIlC89AqJ5Ko~E#1Be)9+dI|5 zUE{cZm5gAdDn6(czCy;(S!)`N6j0Du0$`Tp(K5_ zOFi?ezl#O^U@Vv#B`7(GNXi25lY~c zb)~^qJgm%#rVs~!;G8oL2t|~lCG=8PDuwcsQpAzASD{6La4akkygZaLhZBl$2U}(G z5o}r{TG3M~g$WO#0%@d4AVf+V?Y@1Ys-$3|W0VOhpU^0T6fe9h0<>a{l75RkMK7h$ zbBTx8#5`thR4RFu6Lq$?eEiClXKU*dn}*pjpa>hFR8VRCfo*ic!4(yHC^-UKXNDV% z>+W3t*poYc`^-a*J#x?3Sp!1gKXRa#NuER64iv*qT}nu6w}?@shzbH^0t6|ev{YfK zqzOpq#exbhiNpv(86d^BQU%7vTbk4^rjbPy#6FBNYbr}8({a1I80If=6d9Xk3~QSR zAPr}V%HIK1IsQ}>k^$5~itQ|tBh(vOnJ6%^4C!v<;@7RsP35L6USd<3Sn*l0)x8*9*4q9M+&#y~I?G5r-S$%3q;wrFt9 zg@Mc=1wH<`I$-KTZ+<`%GBVj9@*F_9X%niy_UV~*0YfI{SDW*Tfl!%8B&bogHI3Sc zPVM37$#?_;zv=_}@?l)-WU3OgBl9jKV(-`wE0V<$@MVyvl}2vqHm4kJ4v_?u23G=h z&$teqTl+>3k*X1G@{pa;VY7!V23oX`wsUEu@!P7Di`bzNcu_zqM*wJvvmru(Vr(Fo zWCI8Sld7s{9uM0EkiYR#K{}Mt8ngOFuO^R>v~}>!7FJ$GKUs%v2qBg9GMX`mw4A)d zWwxc+^wBVAcet9T(^|%0X;y}CS56{Wmm$0&QEV~~|REXoiN$3IVsUr8%^h6oV@qA0~e>KhC4 zA&LjUoMGB`Y%l;rq^)h!(8LtW1w$uN$)m`ipzdE= zwWKU1OHzRZUmR7A=9!?oRMIFWRkrw*x%yNyv;LFPr5s3g3Cl;G6LU17}Vd#DoSd5KxFPk)agn;HwV~`=E$6FO5Emu7#e( zh;3_3mEkYhViFs5smpprQOTk^1yB@$f(+DCWwfn~0hDHWh7A&HWcBc+7pkuAiVac-Y7`HZh?wc1 z0xr-baMl`BtKCCr6?9;IDN3ezf~2*jF*V)U-p(+eY&P?_woEh(W@fZU7V4FR8m~Hl zDC$F!X;k^Bm(ECJUDKyChz^;u-;n^7OUMrfh?s^xUlZ{EfmsrH03Z*P^lt#jEErMwQ zX~|mMEdWsJbYy2q4BPw>*nqX}@YHvdHH&k7*=(@NLM=3DE@K)n#5M(!%N!sOE%X#2 zG3QmNKM?h#pyZUInQX!e67wZ>bPtz`{~>>A9>bg)&>Y#D1z zjZHN8HdC!$OXMkpD_K5dT$~mLk;S?P63Mwr;?{u;fwX6fhuO}3ab*|0?8hXM0rou^ za$s5sNvX*&K`Rf#ppv6vY1B!|MLl|O6T@H&X~uC&OaFqduFm%1Q8a2M!5O&F-NbvU2s%S7u)#(l+Ld@LR<<- z)vooOXd{7_25@I)Qlu!E(==b^&ovTIPF+Ab>0;kDo3na^-m8NZlWt&tBZ&6(+XXV);e*#}KePa-DCk-I2B!U=xVk;Fpy#`Zdn4lS)( z`LuVIMaa(e+rNGBC!Sor=D73EJL>e4M@F`^b=2c1HeB@`a8ULGG7^ynKP;UMl{WSr zYd`tQW!HV<^3LAA-+SM?7VN$2*yu2e8?E4;B)FQ1A9(A0NR;ekg+3j|+X`tVoxv1e zn8cJcu^g^*F|G>?YC2I_vd-YqPDVvKdAbxUOCA_8!oas?Rrh!%joy3C0IskU) zcw8mrgnR@8SJ{dV^i+G2lGC`AwNw@&aN*TeB`3g5-J@4HK$|$_8{v}=Z5Zo>_f$jS zO_z!wfmjGtM>Y>YROV9*S zfCRVplq9Vcf97{C^}WUvC;T*WDgiVGoVg)T99^OSd`J~mks}1>B8Rc60v=Tihq%%@ zEHq==#^K4&Ue5pLZNB0q2ljP$rt4Rp@QDZ5&P2%(vPqj=CBW=08bVBOGFOMJ$t6~o zxvSHt1E~bIDD;Bsbp)gm$-Exb*Sp8!T2gw(nk=xER&1{=Uf9=FuMvo~G{CLsDKw^R zX&D)x932^z5KG2GQepu z4;%h0=V;G{Yefug%)dJ`t_g59EzlqygpHP zXg$;oJXPHjevpmz;xbL>&N5kUV4!>3*3I>S}I zsSw?48V4eq{%2b4tyl9!3ZOl&Jb<X;{b4Nyn$MRa}6h3t$0|YA$H38bK{h zUh$PM(#^?-NCZvwz}t3Zn~b>+csP~_vRQbP^Hs0lDlV)=&4e9w0f7RtxMYVU3$5vE zuDgEKeGi;+^6>{BaPZFIaqglNMHulBJPOF)DaDCFK1!=aW?MRY>vybNb@v^2?6%kL zFFO91T1RclshuEh?O@waXb&t6Sw1+Z=QXnc06+jqL_t(Rw>qM!O-^z`B5jo|XvH3B z07?5K;{85BWav;62ZJDJ0ssX*0B~WzmsFHfDeMK)k0rIb1<>UrIT zeGQB-r#E&GL>Tte*iV{jHd>~6)pG(5NKm1j$4=Zym0HAtj9HQvc!sEw7@P_XygizT>5ig=OGNNC zAd(cJ=fg1Qh^#W=ht;a%k2(^ffpfXNZM>w%1VWc73(Z)B6Og_+R|9H01WV8`n(};9 zUdzkbx){4~%ox@(FtoHH;5|3dEmjt{9B5>3RbC@&ul4=sQkGeO`!;&pdMP z>Ma|#xed|Uk%dX5i9X81CUZ;c;ynksj;XcRW}A~`;*AYvx#pYU8mC+Zz?`bGwzAP< zdV`Hhxta%_a-%V}%kt${eE9wU_L;Au{-bLj+y9V5Ys-3+9~40KEWf}P3p#w2xMj9k zJl8VQ(b~T5zBS*u_^X>A-#$6M^WyjYp0sSp!A+_Fg! zxI_bpfN|@z=R4(H=rmlXM}(X*(^5KNU;Ku zC?(GZ2Zvm#!6o&0L|&$FiQz6&7KiUHzzF7MSZLEEd4aT)95j(bpI320O(hJ_qQ7`l z0Gh@RI)~TUP1Tz-Ef|!`Q!yYUo;a1@kxjZXjvJ+CFw>T0F77Eo87v4ZA83F%!9Yl< zXz`~+csPiqdF2&oP9V*-ScXKTW+#zqQIzH~+lrWkJtq~6%(hHXz=pFUQ15H5@Q8-( zytPBx9!jn@Rg}R+Y3F3HQ{!qhh*xU&!@0l@&NJfcU&M5%dJXhl;Z{16zvX?}lut^Q?+IASm%-NlH@kP;q*&2|$oTT*x8` z_qWMBPQXGXDd`JERHi}^vH6V{@apdB`t(OWaph%~b$9ihcEPL8f9cF4vmd|Og%pEr z)X~P+K(i8lUU6uG{OAeHOCr&hfDy@gdccKzd4Z0@54&SquF*r?IhDrzVSmU@jgM~s z$UlGdrk~zatJPVq8kHnxp46HPf;FfkrU#D0P*3pgPc6(((V)_JoAqK?W9spa%OzQ$@nC66I--E1|hu9O`H)u z<_?8m%mbZ|NQH@%}qP)d)2 z+pW}udS!wj&|`#a5O;8p-RFBy8mOcoFtVr(QD;cKY(*TXb+pyl0`083fo>3R>FgO% zW;z-M7W~>;>zy?p9GaP8)xy6PspRnkIG1Lwveje8e-xkryag%!j7|BK5kzTrc5?g9 z&N@H6L)%9le{}7$Rvf`2YKW};h(u^Xj3HYz#z)9tbeN}@|M;IaZbeSD_}ExS@A&ZW z#Mn4rgPG>p67m`9wQ++e!sTHH$P}6syuf*C9jiexgv_v6ERw-&BXy)Vb#Rzg?dOl( zS{FTBq|uD2a5b#Zoi2%x!;8GMfZ?wk0;O;g(_#E3CCLL7)8@?+B@cx%@q0Dp;>ifm zM}&@6q7ec}rPyG&gUm&;iweP13xQF~8%@}*?8H`O>Wk zW=j`> zkz3z*-VyumxrqK>aSbC)q|Q|!iX?uZR~G6JC8EkdZ`D49>wJ77xe|sy1t^u4pfDa7 z2h4cWc@P&>{eMIyn38YA)f*Ct-6+6#Wh8LzjH!v-jVFwZ=daAjh+OX1Q9EQh@&AkgGw3n z=w!x_xF&)Vds@y1)=O&%KD956RSD8M51g5vo}Tp^pZw|ft{NWe;sKeTe(#5;p81mA z1&gqx`l4_}46+0xCN*;*Hu6LjNlP{pH{4?)`A0B6lY(?#Mw@x21T``K6nW&~3Utn1 z;F_$^33NWZ;UBw5z?|(^jvy)>w$N&2Smh!iD(s@bLJpx88B*8}5A5pS}4H|LBce!*Bq2K!?AB z5{eR221p~TLL`V1;dCbG0wj$5oyz>tm%t~=2qPYwShsH7T`TYAf_uTh;7d!Xi8v3~u= z-F90(I@WYHlBCg$RL<@&q0yYfVGNehNcp~d?%Td&d-vjnjnRqK53U&+9qaAt@#_j` zLl$t0g`$QjV#;0S9)*v}yJ89iw_*T(3ea3Urb z5Jf9TM|x4Dfc+vud_)Tb%2bO(gpezrfdOZJWYUgkR*?xq>W0uP`UJ_74y{Pc38mIj zRv>v$DN;v8>oPW;LlgO;kvQopDAY()fdC%W=9>}H_HxKuL;JMpI!zMZg{V>isfFwi zKDLS~bSmS#e=Lb_BOmz-+mzE+78^x$(TPSvf_3}@kB@xv>v#R$+0#cJx@&KD{Z}jn zqKSb78>dgXQeig24{|EXSa3*GuH~Z0m2+e@S|gWv#i&drkh&u}DVxba1h{y24u-As5)!LyjvN^XfVbWq#50^pvq4A z^s@c;KI)Vc?z!P+T2A|&3+lb~k&z*&!-c@cL`fFv1eR{%lm|t$d{VDjRxM&&E`$_S zP|Zwp+!{b>YVbJ&W~7)fxr|9rJNB@8ybiw^_G+Ad99AK!UqJTS0^Q;Z_q-@UL}SU- z10}uoC;^}%wkHaTm=s;{urGjHV;fn7fbw`57E-`TH;EFxTxUcRaHNK_po{Dv=^YLc z3$g@bfI=$Nl~Q=ciPfqCAC@4fF<7w}2En$K!=V9)zLzQ`KO|OKRZfIZ33-DsGSQT9 z^-~looSitTx>$3lRZ*>lieY11F>NV5+B2h;W^lzukdRTJ(#lzkd9lPsuZpH9nk@ZL zkYtH2o1(nkkSIjpjz)3{0a`6boiG$dSbnsHy<)p;(av5|7v658}SiTc;ke#hFO z#-(??=JW%fbKtVhI$P(V@&7o%NIC?nU%e0{&VsK-i+~z8xF+U`zf-Q!B6H=VKf#kh zM5|=*vTZ&{9}55zdhx#c6~vg4NK9Q_2*CDszRA|VaB*8_t-U!_ulp}6=0f&Kt#Z5|ATwqDgB>>rt>aKS#T(Wrcwsq~b*2TLn>Fnx) zr7$rw0w)cck2$a+oalB}Oqz&cd*wt`Br7O|lR_rf*u+o~RSU=-Tu}-!T0p6`y=|;W z0H`@7UYJS*+bep&(p^ld{#7E9eYl1CoOp3=$tSXZ8pkv20tOAVe&SJ=lm&nWa|lL1 zL$374go%>Ov$#K3!Nsfz9LWV<1nmsX;<0!q7S+b3CUk;LSH#oJ@$r2RI`|E5e9ew+ zJEo_`S3kJ=<{NJt8kr!m{OdrhB1;hm0RJ~e9^xTwhn; z@6MD#9E?IqQgrEVLeoq}RX)&&%OzU&6q4BNQxc9yl8T96UNn}j02RN5UO~}$AO)}> z7x3cWB4V^J)FXy)Tv2NzmRsw0OkQ-!-Nzoj%5*|B9uj{o(6bd;rbk8;{*~@2yu}%fhL%x9gPG|Mo6> zui%+T{+Hg&bdycKNxn4!2X0>%_<$D(GtbC7h{U*x8QU(*N5 ztE2YL_Lsfswa1-%23I4?_uF%PY#99*=m79^wIe-VhbbJf3C{}p8Un_UCh7HFXz${H>t#KA#| zA%v|FCG~7X(&;cQQB=AiCLy8ubICz3!b(M(csP`*!+4IS+C(Tm456^pQYON}Hi5zk z$P`O+_}Ku8Vk)7`EzK=As3TE$yHHd2jv`&C1=!K35hEpPL=~80N30jo6v6HgoF}KD&9#MKXbU0*nHPma z7gUAewSY!|b$s${jDG5hQ z(agFrPoSxXRuVH%^Cfz1Ddc1VJb5KggfzU(a_CV9op<4R)Baa(y8vNCPBxkgmoBB& z(b;|e>o4pbSj;x)F()0f%d%zcrL?#E05!4V@RGRvb1Nc4f?Finb}iQ}07OXPgjP+} zroa^)!ZG-jPtK8(u8?$^tDHoGKjb|&FJOA-&u?3`^5F#wdR}<^@dqECKl$v~@U<)U{CLoSm91+BnUzln&)e@0zcA9zz7X0>qd)>8HUfn;? z|Kih5sdx1>8%-kCEfJ}i9ZYL#=nyL_Z5l;wUn6Q)>-JkC9%({S4&!Rp0 z<6RD8LNggFF%5NBL9u7)h;=mv-nAU_N+(enz>12EaSsG(H~jcWqg}SrZ?Wt*aQkw} zE=%6k`_QNO8b8DBTi>|i#FJ0#?_1DhTM^y_6vKtP`F;aL zr$qd0xJelh-itR=^HziOSscKU+dE}RVxpOzX&@873`fdi6WsYhUI#D(9cbok2EW0Z z@3`8_B*0U|1OocN1}egkNf{T$u7B~4KR@o6TNVs1IOe$D zf~jgT^2shKX-BWVBm#FV#H-yX6pP0snGA$;HG_f5BVbZsug|57VL+K*wA*Lz7SpsJHkbxzR^yCA4nN_8bsBr{pWe;fC4iIPI`Ah>R z<j^waRb)b_R3nNqMXEwv+5rJEtjdRG&;nr{j*1}3rw$3s zwJH&RG^7*(V@mkvi7zez-_NvxRC;SM!LBjUJxdjpV>fQv~ z70%8)Eju$g!K;$SryG-eUk9}Ij#^LOLUI(;M<>SGYBOE6F2A>Ce|uNY#Kb6*K3DiGSa=M&*4efDf%_p7^!ZIb;U#|EJzd=JX>U%9 zO-xO6_Vmv-D2{b>cGdd(Cnv}Ft|68nueEVuGuzTTFu>x5>#Om`I9DK=tgXGXtEa7_ zR^wLE_%Qi|@9zw6*+QOKymV=6Pi<&u8<$o-w4i4+91pSc{9H$^voSt7JvIT^dS_?< zz@i=F!yVjq>gplT$Hs@eD9X#GOMVcYg(%*ENPglZo3*n)@mOm3qwtHp<@M*WFVFy zZI`kK)DwRRB`s``zf!7r7l0xPy&)RH~oJHS_=eSITh zupBZcBU20mgDzfSt_fzxo80&2(-mBocc0wnnr!+FYQAmQ!5d(CBBsIRNt2~uqD#Dq zM)P?ueBqJLd!FX)sEtidK)QXFFZc5VskY<0I^?U4_V&)sF79zr85^Hyv`p18g(mQq z1K8#iU~OHUo#^9k2ybZN>on8Tb#8EZiAw`3L=rYDpLiF^<~(2gNDVre1&2cx`jAhU zJ0TW!wipL?@@JO9gqk^Fp&PNWIE_0M{G`ZjIYJTx>sxL}Y_?Coo>x3x}8wTz5T^>(%O^wyfJWM*4N$C&%2 z#sK1#Up>8DlZ~myWP>5cOx`x#vXhqqpfeMvFCMM;_tiQ(xpk=o_1><5-de{%2M>{r zj!icvrfMB+3kU1cooN{wA>vzmdwn?xQjd&GH7B{a;jJLNUX0tYZR5N%jg>)1%c7pH z=H$%KXp=$25{8F)iT*_k`pLM_|6`$uO?B{TU&;Pr^r3MJ-$erbTKl* z1}mEoWy`>9iW!W5NFlK{(0!4?_IOX95-VPt6+V$Xy!9&uES9J!Zr1($WvI~`MJtky zHi{6)qay`SreXp}sbEO8QVRroN+1=O^l2ubI6zCxRzTV{?dg(qK!hqKrN}n$9NaH~ z&o#lQBA?C*0@7S6mm6cUkR})!QPB%%%!ojhDiOe*N|HHMM2+(Z)1sdUg=G4y{gT$i zKkOY`Me~@#a-keiQh|(5*3ury(oujyyc&{mB_Qc8Fw+0`oS))R`Cgf%>nLZix3}K) z$c~SH<*rwqw*Rru+lwz~&X4!Nd4go2V%kZjR8I*+l1_oWXv9H* zLp;VQS&S72s|?A~pQFx%3XEGG^~s&1w_g2&iJ|ea@zFz%J$kSG5BT22pMLQ6UrbFm z&i~6l-R;o*k!k1So3H=MWvgz!eSDPHop7I|W9f<|$DVoWb6Tzx&ui{%IId)%vDF*P;P+1tNq^&@xw=sMnexW~Z<9Qo3dulwR9cU}AQom;jo zSh(=uqYr=Kd1v8ZvG&pD&sR{QEdeeCfE*0KQFeec~*c-5KDefi6_ZQn!=!$+8zw8$HcX2zzj z``R~GUUmJp&6~N#99+Ecx6XR$$*(#8-dk^b?4J8bm-`)k#EQLlYc?mdd#>X&ZLDM{ zaKs~Ji;8F8@J6|`fd|mo(<%k+Y~%#P(ONq!AIx#LKUZ4t1 zNOE>diXvL&q4FZ3f^o^0wIhJkL`|qAwMCdXTNw-#;U#HLK0P6Yv_ru{O4TW`Bw?Bj z_zF$1fZ8%&$VixnfNF(UxKV>ZRLmVJ%3_9SRh}1cj-cBV* z2qg~9L^zR9hDKC~qFG@TRTN00g$|?&qB_}6uSPG6h&0$E5Sm!`_+_kOCfkapI3_mG z>>M~o!g2%Ba^-Ow1ql!OOe3WdJWjm4p{=zs({|MzPd>hB==EnE_Ur?9Wd%GZ1f&fx z_!Y_K;?QZ4q(mY}gvtlf^p=u>kY^Y$g-;QWXoHezjUZA|cW~S{N%(g{hdCpT}{I5s?1>uld;*IoM-EEpahAi^Jegr0~M0ont#N7n`*`gA2L7;gW{?+Y_S`d<}kla(r-b zu&b+U%jV7GS*}P(M@tqj+Oln1OFPRqb2yh8BRjXVYGTW%qler(UB&-6I&YV%u2 zEEBu?`lqLxTQ+Ulxoul-f8QSa>^s>cvySIkeD&bshpU&~o}PNGedCiGHm+Zf>3FBv z?tASG_YLbeaLI^r-F-a^7B6jbp=By7lRqXYhQTcwr5M&G340(;F;;S)CozC_WR*x( zkt)bC=Twc4xBWd3v6=7@cTv z)$#M6-*L?~Kk4pjUAlPL+2@?`@Z%eP=Z}7l z=PmmBdrv&&q~*((Z{ED+YhSyX4b<+=&NI(End|n?eESDmH*KqR)?WLDH}DbwQC1Q2$&`VD_snKZG+FQPT`IYM*f0zt%#F5WF?AZrj_Jyl& z`q@oeHf>z8%ip2|0SryAsk3=Bp;>xl*RHEY)X=*nyFS#{6o@aRDQAa{^Y zJ^kd>4?TX%EqAalRib58-lqbcxrhGcn%>#)U!KGBt z{IIRyN*>i6IRNNQlNTU`v&{uKV78Wqxke=yGy629F)j$Yen`h0a^V?erA##_D$QZp zA)Zfbl>tuv2o9_1@Bo3{_Y>_lHw`c#?@TU2B7&u^cZdnAao#u5aa z!i&qGth7#ZbW@3cY>EuQRTd8BGp@hA`vEa0Z_Q2#ET};Sjw!g(08x&H1Vy`Z*8RqB zQMO`IdN`NMm0}607a6ESmQbHey5rv)8n$xQEKWp9Il~Lh$3`cvyX?CU-SgngbYr)r z%WwO)tFOHD@>c#)M6IPUGR9NsliRj`<{j@@bLV}$BaIzRaJh-G;h`t)x#i9mtz32X zpTD`K-aa-lvUFj6<0Fq=`Sr`$xHVdZDI;#K{7V95gf6twFK6(F|;q5zr_M@x1d%NHK5AWG)|9vJV$NdU> zb?9O(jkEdiS^_)e!4a-tVo#TG@iMl}%nu|h5F){0f#c#cP6-Ughh)TXpt#mCTBs0a zq~=i2qu_y4Rs#5FP7Fxe102{=mImPzYSat~I;z*%3xSy26BE?PQVEVYJqpbYA_0nu za)+wxm{6BuLLCU1$(aL$QmGV#B#_kzO#BO!HcTNeSrm3bbm(BoLdGr+DFjBnB2$_J zR(T~onxH6wl#tfe;3e}*QR~E4RE%?FRff>4DoPjXwqqHMElXP~K)WNMrYZV`j*S32 z)C)!tDr}@su|Mw@qqH-R*6@oDovoV`%3Mth4>yJ^%Y- zpS<)7mvwe`9&+RXfBg^dYp-`L7#O(xvP&=im(TY0_8fWK5oeuy&gVb*xnJD%z{E&% zYHED{1NVN@Ti;CA^wSd) zV{d=oJ703z%Z7Ju|KNMxcjrxa4lL;VoA(1Tw zUizsoexB!b_=3+pci;K34}5gnhV8pQYsH5?{gDs8`#m?_c>5%KK$FeID+b^AmN%Y$ z&bd!+++6d;m8|00Kl=U;-E{peT=|@I+Kd0kyZ(YJuYsQ4zx~G#-hRU^-TmF?U2yJE zM;`g9Pkj2`RS)jivc0Fj`}mWOdG#A!zsKJDjE;?$@M$!s`g?k}tbg+BpS}2opWeKE z>kvEF_3rLNp1c1W-}L)8-*EFK7hlpG89(8yQ$P6e53PH0J=-8hIZx^6m~>oKoZzoUc^ErhiQsb1o5vNjKxekwrNexPZDs1@c?$rt%IbT zD;4O*sJaAruz7_T%blrdR_EhmlTSSS7?(9%lXUWABKIuXTOL^Ti_d-XquaOdc=4&H za2@>azx>O4R;}(}RCcw$?Oku<^0%Y2xyZ7FE?*8Z_kG=12 z-Zeftz30A**suHHcYk=@kFI0*H)e)@>$u}za?&w(|Kh=qeduG?U47m59Xkm)-uu^H zul?xCAAj+GfB7%o`PK_Be5Eg2pKW3L;GPHW`Ot?xJiKF+Z`Yl6>PZV1E+%6=c>k)8 zeE7o~HmrN@3yy8p>L366zu&lVTX($&1OMQPAAj*PU-{BEFW!H@{YHkyd4gl2*|_MF zpZ&$1E3w;o=bwA(=`U{ORnW6jmwfG#AASFd&hDO9yz13m-}>$)U;HYMlXUQs<*BJF zuK3Q%J6`)2@BH(6dk<#dt%Ml9yQlS%ORxB+_x&;RWUmM-7r zOP~MOO&hoGyZ^qYoqlRxPq(*Jl7!3HC>hB6m^pMUUh`!#?(R@@h-G8WAfcfE7y%hZ z=5(9^pn18pQKN2|Qdvm~khpqAJF=5|kqnTGHcFZPUZew$ERF-;Uwtt4=#)u)mAFx`-<}vBm{6 z4>F>-9CG-f)IPOYy_hJ{(MIkP4f0-Psymy9{N$oXJ$X_haVrnC^AHT2Le@Rb!V8A9 zdv$NUh;A@vKNmwG1Qf7{l0nq8A|Mp-gO7W9os%#9M_~AXlk^8)yZn*WYq8O6Tcfk9Z*Z3-{ez3X^0xOqaN8;#tmt3Tx6l6j4lZ7> zX4RS});-ZW)A{r7UwzG+UUb%aeWj-FTlR*C+O_eL=I)l+&Rc(U z!(?N$UhnT3sE;>B+uX6~U9)QS=l=5D@A&#>)(!38?I=L=*V)m-J=^RiaxWV#^?8l| zUNh5n#h1Uq!)blJgIIvQ%z9fVmjPe-z=wB#_JIrcTQM~@T8|n?YBi zUg--WW{IpSPLs@lMHFpRK=hluPDF}fF^wq22NjsOMb1zWULu94 z&=~--LnH)1Ub3aPr+|bR;G9hz5++>)vvf+4RzadrAzB(MnG-Bw7bml>kMj`OI@vhbhjXxTE-cF`sGj*Lye?D+lp zT7R%Tseq?Um5jTBop!F$sD@>MQGY11R9);>A7WYH}@yJzLC4?c4Def)2$_Rd;QcmLY^*8Sr<-uI_}^R}aoIgT&y zL4<`_fB(Q`U;4&pK77&WC^uAF+oz|8I@&ir+PUiH2X4FRwzt3Mf9?G&-zmbv3AvCr z^1%w3lvIN_Hm0Y?o8#MA>eRgyn{An#@LwoQj12GC&ZDPF4nSk5InMK{h>kTd8m+DD z1-H-gJW8W&mdxeL8vF+tT*XZ8oEjOL7#f=X%Rl+6yRN#az0QSFd#$H;^ZLVBn~$$q-`&&O z*Ru%4zxU0n9$Wk9U;X3X4(_rGkIyxyXBPDIYUcmDC)A6or*-{7KJt*gd=6lgS7 z{^ZUNt^aWE!bSD&{^?0x>*!Qb#u+x?YBp}cG?vhc5`WosBP=x&+M^L=`WdCBFGmwS zHgV#hs{}veMJp%9V_+YB5&)0$YZQt4^p2gw@dQP(Xq?$QFlHHDv8DxE!(^=5#7P5g-TYa>5DjpT%WP^8ET$(FXT4 z{YMDH!$X%}_MMx4eskZzz}WE6AEc1Y^xD_o*VDPAW6{9Sj@f_syMKD<{zn!s zTEg=ry}i9PhHOX6HP_tm?*IAjEt|Fu3@q4ZpM6&BzJh1Z)~Rnw6dY1A=isoeJXaDt!n;u)Y zXwf1b@|+y+>g(>BYK&cb)m2yh_(!}O?a)IGUbJ`-7eCv#?by6|>)7P@-~H7;banM! z@aprnZ5_r0J)J!Z2X?Ksj4oKXnE!msMk$Y$boKQ1cP*YC?AWn+{KJ3!ehjkzeg`yo zC}eD+wN~G_X~Unr`Om)a^^19_h+pPuZ(Fo*@sh>Mc;0Ma!9t_h`&q5soxKZsm-a4L z_|qR>J2^J8`|kS;^bKs@yovpQmU`Q_F8kK9-Io9E@4X&}@Q*?J`)gnR@;Bc1zy6Me z+QNmqV6BA<`%!%R_U+&O))fN_m()8JFIv#q*SRP&IDW!sxGw*a6E9};#C!24Cwp;j z2@G8w3{mhjVN+af03-z$X9r9Gdi^m}BtMOH`-K|-Y9RV0KhM`WYw=X__?9mEWc zG!!URh{DjDCB#c}v3gNm(vgz>YHBM+26bJs-Il`sV_|qO>N&cwDQN-oO$S*cT9}+)O)`7u}|Iq z^Eh!4*pz;}gHQ=7w*7>I*wJ4Pl!OF5;#dzN55E!*Pp_p0lvB zV=Fb_uPnu*P7JHBffU;*38{3V1wZixD_%&del972oezaTS_LFgZ6~vkF&35pgek7SZq({s)0$3FeBPqHW~(+YHXQloF6?}U?2?&|I_1Ip2tbMP-K7&s;_C~|{_l|f(c z_KjORYc0o~cKkjE9x$?F$F<+Py57^nL+L-d{D%i0{v5syLaJG~XyN5wzw~48|Cf5L zd++`BJovfK-hF>=N{p<%>z)ViefaS;8$bJ@PyW>h|CT*fE^K`k6n<)=4ZNbtx{s{Z z-QD+s6OZ5Lpslyvc+2Q$lbzKA58i+IZY%i0-omAe_>PLU<01ZfjW>uOH%4MJNDJqt z7EcoMUsK9JRwb5$oN8@def?XW*!<)J4?X9YBM&|LxsCDh+pholmaRjByX^M4k6(P~ zkw@;bd=DP6!M|)HaS`V{kb78s2aag1cX3J6!T0voKlDUTXWNOVpTN^)kF0(0p$DH> zuxRNc4{x~P`Ws$$?m2v|rKjG-({As5`(LknZ1cd9#eF^X=REI--S^z1F+IEX-c`T2 z`L5<;>pS{-kO~K9=SCkQKY_uA&cns{W6p_)Us6x;sjktp!7&C52;w)JFC|_~XaW#p zM2r`K5Dk3dN_35gdG!0!JDyn=FsSG3>N(0A57En3N1leLj1#c*+ML#65s>sjzzE8C z#S~`tC?+q)CfWiA1DZ_=bKF3t4P4$Ux?#h{i!S;yUyvFZ9$xdn+LgDhB($a)6APCN zzUFnW>hA9H4Lp1yWUAR#uRXM84d3=S_^?AxI_0?D-frH)$NMTaZhZ3N|MJPro3<@k zwD{CBPC5U=3-;V|PyPYRwO3uk9hisLK61%dE;;Gs6OKRb*dvcR>fL|)o||sE^9TR- z1ODkj@8E*Bz2hzHNc8md?zz{VEb$o#tO&SgH#X83-m!JJ-B&#SnCI=j$MVe^w=Y|< zthK%C#FI|A?UviQ>-x$I&RMbh9$eczzHZ%DF8=BbKf7he&e2Q0`pvV>JJWBydac23 z0t*ZtD)GU6ZJzLAMaK}_v1LndZ~cvb_}>mZXupYxNp8Pf`{SPs3@o|-7x(@6$3MB? zx6d9M<)4ldt7lQ19NR-YxxxPF)WqcTf9vQAUU$LnyREqEj=Mkosf)L5 z-A0nW?&|B$dc_$7gNysR>-XLF;Kx675zzH|*B*N;Kl_zu9R9o`xNY?4BM)Ep)ywa^ z{jT=Tx(g~E2F>_k=~A3c2BZa{6CZIbN+d5j8cHIKUh3wb@(}{eWjfbF8mXAG<^#(? z=s1(Gjh1pUA~NBDE5uXLBv)vML$Re*+r*SkAz&~TS~ipzm8uX03c8RD)I73_9H8S=T2TnP5V4dLpOQ#2_E+c7lalP`(IGrv7C8u@Xd_)%)#J;|(tEAYc$YG0*fBt?I&Qqk=AeUB%hICv0_3jhSKkt?w{B+&wM>jnBcyn};2e~7Z`p8Gg%*qHA?dXK-%q!mfy63&}j2&Ay z?s>!^r~dxy);+Rr@_(yEV>rJ2h4s_!++|-J><3yE@}4Odym(@ z_kXr5>?T3H_zkal_@33*fBVXg*4{OD{9=6D&Ypq(O%FYG(=|WoXz81oX&iOp@fZHp zTWbq?c!_-1!rqtv$s2i8{!8!wSnE_LR?KTypsne&UJyenD_!OuV{GIE!oDn8r@#!ri!Mb44P2s}2;1v>}(yV@*bzpe% z85H1)=mktV*f&ax_e)io(3BX6#pu=cc(5dCdpST!h|rnv5N(cUC`3rW<{K-@1-%HJ zGx%AvING^canK&UXP>z5Zp#*gNyV*5h#bHWDz3@P5Y!G83y6fCf)uS&mqN7V6N?%K z#FbcH0+l9>@k@Hq3Vp#y@>aN89up&U@{S)f>4wI`=i_ zzVwXKN5>`*Y5VZ-g07kyN1%DZZ!~$|J5J2KT=qnKBR%_6+)hCm8>jpio>LR!`yI6R z+ur+*efEDAEk8bV|BK%Lagyfn<{kIkwQBc0_MV6rxEN!j3c{E}KWb%7O^wAH|6=_W zuYSd=fA@E|$^7_Zk9_d2-+%Y*_wya0yKleyMK3vph3mkA{!2dp&HL_J-M?r_e|P;g zZ+!I`=bg=+Dm!3GSNGSK+YLa3iZ z@0B6wpvzFM45AURWR|U(TU&~fPfnVMcHNU8gS@sg?5e|&mqTd9(0jzeW2+zfr}uss zIN}M}xEI8+Z1-LM>^_P_r8xZ|JS)?S~Om|i&0 zb@^pCti1j1p03`PoObeC-}c9YiwfoqV)5f2`RwK`n|}EHYmPthST0Bo zI$$q09IyP&m0S}u@P7N%=Ouz?CngAh7)K4 z-EY6Qzv~^`5L&)$DK}qPjO?@ba;|zFeDI;Mp`mpTKR&d5C-*}n1RY5czXY)svLVfs z$45ur^hf{wkN)Hjc()i^IQ#6q_kH(1uwla%9tK-;|AUy$?PTCEC}XP_Q3~-8DM(r~ z+-UlPxBT9TCmh4x=Fvw#r!hVAf%kr(y{mJ}rY##bY#JC?G|*T3{&%l@V%?Mdya#Qt z|IL5&yQiOVYHLRqo;vb~LzgdG_7Cs=z?!v>7!M>0h7R(=p~M4mLOp{_&t`nVSC|5( zH2T+1AX1$J@Qq5w5RKZx*vY4W)d3PcA!|qmvly++;B*&g`_f+!5g`2L|DC51002M$ zNklAq6QSa&yBxFN6oN1}708Pnys{SA; z4LQ;zi@1=mj14ela5~g@)zyDyKnWG z_YPpxIAy8&ONeoi5*7*gpoOYmW3W%N&2=NDBx53V)oRWWa8_L*rD8zY60=YZaV_GX zXp&LbY5!L|7(VyqAhQt~ag3xQQqfeYvZEpN~L$h$t<;(WoW8n}?K9if>}c!Vv}x1OwyjGR?XlyDjYHdZ z@a5pyTH^($pWL~qcjvY(^;$=>F}-8U`sbbg((ixy8=D{A+S$fSLn36MuQkHflMZz- zaM9u>mp1fz!RDX@vQz}O*Cqwl=9EeqFWaG8XcpCWT7ZvAVnY{iXfq*fAz%uyMH^z# z%zhyjL9+ZZ4g-=H%OEufU;|a~WC(UhOXsO-B|rg)!lmDH;i^({FS)PMLkbd}_5F{CU5nC1c`Af+G_FgqcP+@fRN^oE82<*>2P1*k#}Az74;3NRc4Bee=$I&;DY zBV&W!!bXmj^z#@p#Pt&J#GjiGk?8-h_vYcARpp&;oq0YrQw0T81w~O5Los79$fzjd zfCe;iN^D~%rqiA7B)6Zw-FcF>W4fc!T-)eljA^5$)g)>(4mXJ+q96#Os7wL^3KUdP zbDeprb3dQ&yML!hKY6?XC;nic^V@svwch!C*Spr94FQ(;E0jz0vY3Q`4MbKn7D7Wt zREs5dW5iH4fP$UY0ob<@)Qk-Tz2;Pv03^Pkrv?mvJG(%fNH9E0?c$(Us4?^E-Dv{DW<` z-t@K0Uv{N;owT*xaoe}Pcjq0*&x_%?pu&q%x14_ZrY)y((S{pOcr}wMC>3vWo#pF% zyas@y9zBFDu~2EQqgA{_S9)voekyHjoZsFycHqDjKlj4*$8FrTYY*>Wc=jb1efn?y z?w-4UKn~mX(00*80;R~+uCCF7Ij2D{T(fS~MVDQ|t0Q;sJ+R@}O;^7BWw+h@-*{u% zV~^|@9T{D-cxY^7i)a$yXT&V zAnL(|i@M;niZXqd(;{eJ0})cy6ii8?SOhF7eUb`9jgiKRNZKGntU^}35-TxS*^0(+ z{t;GBvjZ4a#Y-gdO1;fo%IRiwZIYLLLmSq4js%JA;HJ+dnlK9cF4<_Q;Ka~4!;6<; z2ObT^8UQ*$%CzWAzXsLTKiC7=_I4g3?O(s{sIBL3 zuX@>e7o5X;N;z!cPW^3DKGsRpTGhK*B^8AF&j5_^gzwn+&Ru&?Jh5YRV)nML-^OeSYUiAH_W2i_H#N&0 zpY|Y#Z`pFT=LBXK} zJ$}Yau$4cTxqy;dqD>U3LIpYzr07K9w1tT<5HD!Sve@97EroEe+NMSrFaS|4J6p77p7(Y*jFAwo5^A#A&So&2eUGTDk;4I}0C)ia~%vUDNTC z6opU()~bfEWiG#PQ+C_Zp7xhtcI>6+9Wy+@6Z}nt6bsg(#SbO6NgXS&^h7+X$ z$x8a8LoHbdTBcB2e2j-C==l~R)nJud&l^2r5*359V761H7tT*^KKIP|?ylL8Vdg#H2_CLO3a&&ZTbbOkd9`^GxQ(l$MSsuU8Qi_DS8nY-? zgh49uEHB>a?eFI;%*<(!zmvyh1_sNze0>YQQ>jWWQ8E;)OjRo9k6OFN@6W`@n5DP3 zUaqQLJZm{W&x9jpIoQ|Tw{-RDnOWaQMEb-jyti~<`S20Ptb6>y9Xzm;vp6c>7$HM} z&U7A!J>w>AP$tr1B}-jS(L7p@I%E_kyHO}ZxCm*YmCVFiR|80Oe(nM zMSe=8*6^gIk-A#ki5KEH8Y6t#!lE>kD*S`2rcDY?5rl>~R?HGfTa2wK`O~IQ{;G3I zAYwLK$_)*++0l+72Fk68gF8NFo`25T7hbw)?W)CubA%Sk1w_?nK`%Q3s9I9R3@BTw zCPa~)AcKZ=H3EsvzVL#g4!>JRwWjLQACVP&hK#AVtiur$(ycz z_0=Z5RPqokFIn#7deGuxX31*^MEv z7dDj)XMbP?l3MWcWFErdO$JzxsA5{Vdd-@{&m5-4ST(EZmOP&w>~ z!Vt-74PnYr5AGYk28OaUfe_~_UscC}v=bnp zQ4Nc-VK~rYrTDI&`ttNTfN7M8H8mvGi1&yeW8O0VE*O|dvY?4Eo~=oBLDP*op$K9^ z>{vz@Q=+!`1BvEDU=htsB&~b^azKs0QD|)9tP9V5(;xh&Jv;YtwqW_v;blvP1_l>% z*f29iYyzeff##g(9M4mocJ}GKmWS7pda=E&ZDM@#;U7HO)!D@zXn*pycfYUuy$KOt zWQ=g!4YqIJ0p3tCIyS-+Xbi5%Nr)I!#uB%zGr7-I4mzTGsxUme`?@!6-pHf06TG3X z(>XaxF)a7=EV|>a?|$(MU-{0r@7(doqkDI5$4=aIH8MKEgD$)!j0Kqy8nOng7K%Me zix`UIu4OBh4lf?&qNitfoa}XQ8=DqWDN`86)TTH^Ge$6pB}zk<;2$0u>KhoCnPQHD z4jlgt@b14w<5L(LTcIkaMh@&BMM>VkcFJj|4h%2l5-az|;@4ST^VZ$J;kXU`W+=P5 zwCxLsBTwDzq|FmIH0u4-xJR>>l~hJpy{%y4Cos&~nH*zDbqbJ{Eo+z`xZEj_wvi*$ zZB4sQaAAj-lC#yV+BC5$DQgeBGH)n05G8V`1h$0(A?P7uEnETcq>J$kc1+Mrh^k#O z(aTokK+F6hkVH~ZRf_4r&;p4*!gzK)iH3In`NLwWE=_f}42;YK(VX%XAQi4sij~t4 z3^|ruF<{722U@sPQScU-XxOxu`)FZ-T5GY0nt?;=YYYdU_N2F@Mr1~&)|$C5qyFxN?xmL`As3``;y?vaMt z*vB1zbmS(oThgZ*AYQN82PHMm)&J?ax#6X&uKm!*KK1VRjqE)zH8sgF**`qkKQu73 z_=p{o+h-=mBD;c@CQ>(3;Cu5Sil80)Eapx|(?z+J9Qv37cZ=$+7@pE7b4D5(uvXIt zB@{TG3GVL)oGd`%jEs{Bqfkrpk5 zlS9$7^M;hdn2KXhAvk)KI>q$x1WQAi%E+xm4gJtVoD#heC;RY{9T<=bfk?$Fi}V-D z%9Qg#HYtgs6`Twc&n2%kbfPA+=LXZzfFA!rr?N#^iBpzJsH&;Zp%A2kl_({fL#73; zp@j&67X9oxm!JaU0?=?lqDTRTHQ5u}V(6weC@s3OwJ|NOphwD##ex@kL!)*Z@v>nh zMx!OHC6J}TV%M62s|Ku@W7_uF+1I`Fgv-u9b|_y^&`nQVT6>;eOt>i`8ixZB=d=jo z#zI(N34@1FL?2^|MMd2@Wn#^4tZS!f#i{lJrno7B9F!$RUo#_Hk(=WO3LK&=Uox=u zf^+!7L_tHJUc~1eIG>xLJiS(;(Tc7bg0e?6q$dXxKgAD@er&1}w#=irJi;u6R{?zQ z?(e~MZod6XpZUUdpZR;FjIJ=$^Bg`*PfqOF^*9LKynhK&9qpXh^-H)8wVm@k$(I+I zA%sRS351`bWu_1fy)~D)GYSBdZpR)cTIsfeCs4I06a;Auix3#a@4xi)8aa>>!j~K& z-l9cabA0MDz@?(3qOC-%IKb7S-o7E` zki3K~%uGZqvTSc(dE{Y!`h!;m9ONmdW}%e`S0ueJsYXsPz%^p4RT?Y_r#$9o_(x4- z&Nx|SklzA^`qUm=Txg>Tdq3sn>2&~u8rT3!G4PsZGLm5Hs!s=tO|dVUGt%MTa&64% z%06bDC^Zm4(JLO-*x>LY3o7WD0G9Qj2pGSXG2L=Pamn!D)>Aj_-0f&R!2vIyAeuPH z>m%Y8+y@Wi2{-TslO;n-7Y_|Xf}Y{v#z$o5CdS6cxMIRl0q+o-;DbcaH9p`6?ri%a zM)5g1Fw-P1VR{O)L_iBQk%2|>QBKGba&f_`|?BO#j-tFm2*U*%clN9(%;G!w?w2W%NfR~VKSPq=LnGy3d>Z3j^ z5QKbVFGYXx$$VHTMr1z)j`ZAzlHapYQh>1(AA<7J-Mouh)1YMCkmnm5f#KUPOhI^M zEy>6P3PeX+PaiJ;iD%G0{Ye3l0g}G+fH{jAgqCJy8svKfAYDBTqY)87v6((b>f}PU z5}CD4X`Zx>?oO#Rj>5?^%feJ7wLoBSXCMhX9c2&HGa6Cb1(E$MT9rz6skbgOiGT`i z69A|WTum~pR1m9#FfEuQLMDpp0^x1fWe`zj4!D@sN+jGGvOwj*M9Ib1I0b@9vs8)g zrm|C&B(5G3FE)|G$`ZqN+g$0XEIGj+rqQQRqjHOb;({ZlAaV``)-sm1|E&cJJA73= zYpsk16848W8lurP{Xi=DuBFhL(LZ`ni>z@6c-?(tghE*N{7sZf$!iu2v z4=uTM3q4KH4S#|FJT1z&AzAFqGv<1lbf8Og8q)==?x}}lcP(n{JIYZ~TV+CQKqcng z3KD}6twkY4O&nlHV0a;M03$bTRtXMK2vrms4Se@WqlA|R4jClRA(@&|g)joKK*553 z2|M;x2zVzS)4Wk**IoC1<>K>`aB4(1wxry#w7xCjUuTe?fSWoXr|Z;z0wc}&va0Rv&bK4Yj-sVxYBuuVT@A{4LM z6X~VA|Cl!h@!mfuRNn_5h!E2tVXpid0LqU3R`PZt-pHlyv~dJMf6577_BcNQOiz0o z?|j98td2Z--QwXvL<0($y8QNSx9=yJnmYD`<1i6Z1rD3eJ%8)+4Te2 z+B`Zjjw#^6z`-*)$mzo$9JE9CfjO?DgZ&w5VluQ&PfB2y$#o`oHG+s8 z!NgJ$$Q0J(&O%+wn8pyucvD!+>mTPhIE=0`4KSPH6V`L{-i6i80lL>Hp-3<@Z13j;Zm(7`#* zNQYuRbK}8D91I&IB{u1O^qDpuN~?5%MwlaAny+Nubjvs2_Le`Ho|;{@`tZvxyX5Lu zzl`r#^VI+S-+SOs|K!iV|GfwE8UPJoh$*m<$Xn7nc>Q;jRw3%>pO_bQJNVdx^J1TI zvky4~vB%J(oUeg{z36eM#Q3Uzv?O1+q$6<`3axNLg=h;HOwj|g>!gDRdpdiTEL{rp z%M*qv%L8Lbw~tQES)3bA)$1V_!*-)*`iL^EPB;EMAd|d zLfN)7WSW^CB`GKOxZI*jK^6EMqKGXG$fByr23-WyDp-RfEt(CLpcf5u5oBzTTjasQ zk;3?+ARrJlhnjLzZKWwB;o0n_PEd);Sb9Rcm6{^fLY11ZfGcd#OsS9?G}&qWh#`Xd zv2_L$G*|Ml%_g)dL2)g0@{a73vWpz0pK~l|1QAq>lRhY&ED*zM*=;C6on0Yi9eNQf zWo2kMr@1T?*=UK-Rw0hmjb%49MP2EphHb=GG$cyKXz!YDKV{?Kue|b%b%!sZ756r< z|8xTekxnZlcqNwstH9bC)r|EFNz}>LryGIv@DMV{6AQ?YCZT~NRrJR&HWqbd!Kc@m_+ zFy;vZvH6(|nfk}JPAOD}zX0$ipwEBg<71Ei5Ql3^vc$!+As0WHRH&b{N8wA9z1Imz6!#_Es;$ z1?{@~1~^uX4pit^h@da0PQ8fZ)XBMv=99eh0puSZ8sIK-rZ&7|?E81$d(x%n^X;r| z?ug?d-J!)Z5AND=--GQ3yUlexv)I*zGA87hS2+!*2#oI<-O5A4(84DB${3jlCkT-6 zyuwVCE{%T_tLdc~o=l%MM0QD;?F+z{4<$=_ssT1QLaX62X%a$}A*fOY_?M92D-}eT zuXHf$N;Xok7(_yWl&R%NY@O{QhJYkoj2(QU4JgsbU-~b=$&>$-XfO&f4Vn&;SvT}+ zw?*GQeq|%eFi|jTP59D6XVyx~LUF-fEg}Z7LsuyVT*APi*~R@}X)yIvLA?@m1~RR> zAYm#)0zVXfB0<57?BIv0_kBi~h>54IwV&q#Kjj)nnk=ne0Pu^lF-g20k)%`HA}+R0 zOo$NM6Y%jD*ZVrw9^UnatIj@c^BQsyQw?DLj|GoW1AN_?%J}TFYp{U3Ij3ng5EZE6 z1!G|pe5>u_@PN^pG*v8sfkY((njMSM0K6&AshQ}_3^uLqsi(2)+@0fjc+iJAJv!Hc zY*a)rD3Sa;^Qg08B?!XLOVS|1phkb(*O|Nr8vpi3as;O#)Q(8tqjnf()5)9e{O0Xs zipyVo`BlGgHT9$jL7b*@j<{2(x!IBNDKY~b*KORmdCL|}r~33YZsVVrn4F~JvP-!5 zjT!`_V*`(nes>8a4*Kz$?k)u6<}}`vn7Nyh;P-s$Ue0!7&i|N4)y4db2AM5&;X+SJ zWYKb*yCqM}@LDvcFvN;IPU`BX$kN6ps3;}m;TQ%4=AsN8qz_-l!ZC~&lR283U9n_o z|FU6J;tcwZM<3x!e!QH3NB>!%KQ}pf-yQdG3(TS^?%ack%#4bPRoTl9!dhA>xYy#esPO0D|Z%T7w@w5-Tk&3n* zj{Ra7!(*}f@WZdgLDxM_@)TbE*MPP8D-mg zxJ1{j^AUkn>P=9d3y@II%#E4X>Khpl>x+knPCs+YZQuMRW6`QrOV7LDEItYYE0U*B z-levmm*Tax_e_jVap?mk?cpX&LWaSR(4`A`b`6c4M8Q(YlQ4}V$~lVdYX8Q~x9-}p zn;FU0bD#C5H~&`O(BipSgM4aol+zxdc-<3GO*zDW4B&8QDGIzIY9NfBxIw z{Pyl$+mASM9iPR-R$Rr|yLZnmH+`K{0*3VTcnY%0*?FWj2`Y7zaf?r(@3wxCp!74i zC_yGd*uzjK`Xn}-3uq1iN383Lx#KH(FVaXOU{<5f4LY}>a%BHC$W2A?u+BQpM06r$6nqsMw_Ecy& z8pHuAk{Sl7DENp@s%p$F4WShi7ZWcz_JXp>70iR2LxN>MQj z=n0rnMjD~mk}XQbj>-`#^NaesJ1*F=@^x38x^%d&(n<0DXVy@)gGOg8>~VO*qC6CK z`m4<35aKjfXE$&O8_F`bTo_>mbd%>IZW&H=fs8?69l@LK7iD4=E^AVUc@wUwM-e5} z-WKEtKvRd)OCoCH#Xts6Xo6b3a?c&(~mxD z3-{1_2gk(x```MuU61eY?C6tsj_Bl=kwOmPK~HDS5>d=Q@DF!n_ejUihyWNnz#ylf zVFAYAVvFPGgS>PWtJ^y`r4_d{4KX!zU<3diiZ$?%Nig79iMfkBl+-eQEwS;U7N;E~ z6ah5Moa^LVD0oyA4Y##4Owv=UIo1oPa;k+~OA_~G%C_m(fevDTU9qRE8ElodmR6Sl zA?SKdNGZuBwH9nmMMN!`1#^KZWW`dR203lgp*tD=)K@1GAwWTORb-m{#V#lzM6t1s zQ1WS%1f%|>l%=*s2$t^3TXp{_!J;Avv}6=h0j!T2Vuc(EE#k7SSP})mOefkq`Z_vJ zJ!bK1t~_bOQOhCi#P^3RhwLs8lupAmA#O6>(3|Xq6A~NsC7ixj2}C7e%#>=zKOsk|mo#OC|)xsXl0Q>LxPW z&!S*VD?D)uh5R`_cktxzxz9fPQy=>HdNJPw{i)L?_v{?a`IX zm-qJfa3g!2={YdQ>qthB#1koq!poDmS%vEx92vl!=~2EciP1?fP9u>4P!3!ryjQENH}n` z!x^00Zol=mo4>)$HuM9+sYFq}Mh?s$p^LYtd37|bgjDh-sN=Rw#?S=&c*v(isYiX_ z35+1P2U>zcE(VWaMA<@zwf@O2KeQ)2SzjkBv=_O-_&TnxFBzzH`T$-~6`s|E~}4-nmnF z+;88Avpc{Yd-SpGy*=%tWBa@X&XKhuD0-++G=iNciRbfN8e{*W!Tw%id~|e@w~LPL z-+$xvx9oguM|C^;V?q-pM28;Jg+?o_u@G@q+WCryO0wgkp1Ov8TN^{Mv95>)FaxRt zoNLi9k;wY|vTAD|pPIYyqVu`Vi#6}K{jLxH?@#R9xr4jLdb{}|_{hip>Thql;cKQ9 zc$znAI;AKaJ2LSi7~~jX^j2IOojkU*MMr^6(H~}o1cp*yDoWt&8c6I$9hw7Q>Nk4o zO!yK7g z;yE_NPY{gdl`m##?bcbG2o}DPLGJ81Q31X5WgjEC`-G(wkfy#Unt`B5l@wVaQCy|S zP@JD%HPHEU7q9!pmz~1V3sC6tLlzn%hAFuCE!2QzC-ur=c&Ix87pAhd(sp?&kR*&% zBMX{}ri^h|%gzmCaMU4qLL}%!ZUnYTD}^}NbPz@?Q-d81ROPL9Vi!q3OsX({IZO~*WZ2JjT4U_7=Qf0S3mlxPygv( zm=8TYD60(H5e8myg1qUo^3oe|h_`}JTgIVZQ7WUso=#fADT@+#ZZUPC)CidGTvmix z179x+qkCM&#q;T&rGuBg{G}b-9@2Jj_qN?1c+*?2`}2?7`=uMd@wfl*_Ba0fuYUeZ z9gBK&jq`@khnw>8iy7peJ?ssDhO8qJm83!0GS3l5T?^U02to`pVX%Yh_b4J2>`%Lt zsS05>i3HPHY?dBt*EU@X!-170%~wY?#$MP(b7%<{qIPhv4;mz*gtmB9ll^pL_x%Sf zKUUJInx?MR4cSfD0VA&Pg}8vK_tJl9Q#MKYuP|ZP2yEbE5Y=yZr9&*JV?6}G#b}jS zC?};@16!zR&%9l`c$khL7+WI;EUJ$vMhR$RBewKgyEF#I{gew@VeI3!)+U)V#mGP{ z9K=!>RkB7b|G~pORF#&p(+b%{@2tY}P!m|?Q`{|Q_rV|PXn)>WYkuYDPusX|1+_oq z=*9SgOD2(qSEO)jNqTA(&Z;J?ORFyX0yg6VO;iPlD!U{#WmOx*%%+J$$gqTi7gM8s z!!`AEu#B9L0@-w^q=bQ`B!Z$B%Fv;)$yOpXOgMp&QflmgVpAv4dG1i>S>i3Hopt<) z$Inbm-g*1IfBq-$z5kwjXZm?c3Ye)y?B7Yu2pQ#s?Su?O*=Q zw{E(TmoQF@9Qg9*ues)P*Uru~=T^)3sAKN1HyRqHvS})sxW!^gfV63+m;t^ys4^51 zU>Mm<4?MoqapGgWj6ddo_C3qPjXFG7a`i7<%_SmI(Cy#)?t9<f$8cL(J3mWLAexx;+`JpDe6a(bv&XySB`$33Zz-gcO}B$9#gc%+D5| zq|aj0fF>Z!JYX{Fgb4ImLMk1~;7SGIflkvMe zP~^~k|NhBS&p7$A7hL+cfBoqlJ9fSIPyXVuZ97gp`8ZA}KKg@4zI5#kx8C}7-YT+e z#fq1`^u;3w#<>94-{0qFn|gc3_mBMEe|hT(TaKTcnt1hVU$cCvkI}P0p4n^y!pJx7 z5qfUXVXId1#d|28Y=O?VBI}=qDM=IdTEZQySW*=pZ-5*i%#0 zG83m#MKls~ia?_&EgVFc$EcTrNG(}PnpUO49pXc+M`qep1f&3~%~uzUPM`PeGta;9 z+|U2RwQU0fpZxe|w{6>g##yIwdgIPJ?!5N%UmDpz(lanPKcjh-+yw_%h;t*N6=irO z@<>%3>QWdL)vu5y(<1=sd_@UbrF0pDHTCM-GEn%1XGM*p2S@}S*N~Y}cpXjNvKKE_ zp%n5sCWTrxq%>(#l73S-2+#y$^wB}erl65QpG4m@=QM~+LOF35szph`q#h+21>J2G z<&&&ugoiamX$1ymmKy+#pdv~SLPU^G2vuE0Ox2`MA-GZnEQmmaB&5SE>V=5r72k!X zT$GDM+An;q2b(>pDpUh4Fl3$l#0c`UIY_`LTj~m+SXfJ%=JmsC59@i^rN^Fo+L6P9 z9Nr7~!4fYzUcDnKKAKd+x9$o$NeAq!*?!SwVWfCNAG`dXlN;4)ZMlFafZTT zvA|KOIK9F|0D9`4f6SI?pQl^Gi2e<#S}24zL=H>932dFWydz9VN2sz9by;YLiR8f9 zK!7{i$3_pF^1O?m_w74B_x=wrYVX8Kpf9y($>ZH1EmzSR!D4S13#!~eLZb|4R77toC6-o&jDVq_UN!~5QurRKibtUx zKMMq8wIbP}xT@6)NWu)YLuF%I2E&$MYlDy0EooyW%UMiT z(N%&~8f7T1r{-A&id9IuPK~k`0YIJ2X&HE?UK6QAP6GfJVvq_D6pKj1u7m|9jAS6w z1=W;3R2#UKBn92Fzk3S6VmQyW4|N}W$t6c$e*Up5mhrv{sHV;Ke}17kQq!valwK9w z6@huOrJ@qqsESxE`LY8H+!3b1rKr|aS2vMd(G$o$Nk?__Pt!RY)!5i;Ndb%|xXWJK zy+QHcU*6BUH&B1ymz?XeC79 znlpEaZ9IO{O<%gDZ*b@ zg0T~eTeg9P{q&#E@s3sQ-{CO9qc2vA_8u5H^~^Ibz3hU&|LnEheM8^+#$As-^v5e# zui`209XodH*|ocG@gNWUU^Q-yYus4EsJgB%muFAcIPI~)z)<>urrWy&*l8DkF0JGb#ZAwFQl3r~5rMEOC;$q;U1V96L6 zMbgBL9(c=Yf0f zo!~79ydET;jqrv9yLSHaC51Rd|5yV<8Q?L3$BR7PXkFV=#6VqZyhL*-ef074zUK5iFZ|3u0Dx zS1VdemV-P$<>yADi2xR$&~$`wX|;-23W1&Mp$wuL1`!_03|DfB@S;Z%_pEA2mFFXa zG$W%M*kT5eRb5mzDPg-5YNt%2Ey=!GH2t)}QAB@;;Z!l^u*maKj%S^mutVNZej^-Wll>PQI_KkA!+mI+&AZxJfWL17#buZq_*rDMmi?wtx-@3>L6EM8nYpX zr!s~*+4D?d5(f`*bK7*PIuB;!;tXYP2a%*PNTo3jNF@WI6B$kHPZd(H3s_UGRB zm-pRv=gx;8ef*JajGISo-uSYA{rb=RufN>2>oHS8UN8tU%m~xWW$Ef+2Dm z8CR>7#IWVAc%FEhn+jj<1Lq}XUYkp%X{T7Z~*=;_$id5BFKy?usj zOau-@4)W@;j*gf8hhJT>dgT}X>eG*X|B
    9LVfNH1N!^mV`U+qZr3#_K=(`L^~! zZU~9Y*nj{|Fo&NLIgM$hi54#LbU?}-#zg_8RMw}-KyD5Bp{1R@tE8?(d5i|#guMEC zFu!>*?aClgleFC~ znoyw#P)H3Rz?ew5n1!_L$)Q-vI~>6Y!QvPZg$9^k=uz~QuBiwGsu|o+FeGFnT|&-j zQIt@iEM-keJzy^NjeU$xK?9x?bO^Cj76)EgELz*B=KO7eQTl1LXfcY6at$MlDbAV(a-R9k_{h`<3XYP7F^gQ{Ar7U;t_+J)Q@kmWdQCmamxD+5cXV`3 zjqzp6sLXreIYJsafT$F(VEsdfSsSy-*{QbKiE*l{OO4OW{o=3vGVb`w7p|Y2oOx(; z{NV>4otv5I?(aDFqVswN`oDVZjo6&`NI?LOl7o>EBfqk~CaM~3Zb&r^C8do9L$*yj z;C2B0T3aczNSA`*rnJB)EkZo`<>_?*gPt0xR8-yknu3{5L(%R^BQ?3pE>Ko26wRZl z-80d%@8ImlZW~zrAdtzN?kTLa^K=T6dLBO+-~s%$x&FRhKUWG=cmoU}DFoV8T<6`_ z44+;4#=h<#V5evH#RH>j*B$c*Z+X*)KKNJH{QWf#-2a2Ud-lPP3zw@7Te0=Rt^ejX z-f-mFwNq0*4M?~jal~Q&_BY?i{bP6Eb>IGdBaTMx-5d@WJB!+Shq_5-T*;Qv>+b04?}9SuJn%xdv!eO5m2#(3*EyQr(XV{l1N+boCqjV-yW zovW^WeM~NSYLrhvpfk_nF6!;>931S*X$H^cVjjDzPp0|uqTXJn3e448m(V_ci^6692}B4s}CA#D=B*veG(NDNB}ub*#XK0Tnt|tJ4S`965mD;OrGw zY`OaRCqS?@jteM0ixP!s;fDw-j}Sr<5T}GmFbzrz$$_L4Y3NDF1ki{qSw+dlwAj?M zuP*8v;TMWxhqw?ZmFbb{1g_{7LCOO73sQpICUzr*z?!R(p$Y|EiY-7j2p)R~oCzFT z7hMu_He57?;KfHJnoiavV0bde{m7^9xc<(4^KH`-R3gY^cvQK#Sw1cjYf5Z$P{yMa zQ?uu8S^k^9bZ&FuMXZ0(Qo_JfAtkJ$S#*>Scqof9MF{C8yF@1{a~x_@%LrvOd$_n! zr35}?SoO+cvmC(09~&_D?cgqGavn4Zrp7 zw{UKSkRDpK!n?Di!)b`w;lq~x{Qv#$`QX{u*!U60t(%%0=jpY~Zk=vVPK=#&@z!I{ zJoTQNzd5#JSAT!+&=H3nbXKKyY+`zP+3J-C_T%wI zJwyF3|Mzd4nVI9QDJwUu9Ua*RV%N~X%YWuekh()e{Hy^mKPI3!9u6Uw`iD z|KZ(l=aIs$_C-rpE#p&E5IHzIH8M4Q-m6~vtSeviool}I&>i=5c6G1baMW?lMddt1&ed)(ll!6I1^=>TwT;DDyt!#+aDmp<^PJPP4JMl~|>KR0{w z#?_a;_2hxRgpdwYR>`-5O~qszW{Q@5YeWG!A=k7=qzEpZzyRSA3TaUiNh%Ma$BJh8 zrBXIVV-XCfTmo-PH8g=;)gmUxgQkFj9l{Vtdr(`|;Uq~QQ9)j|q!9V5MtB=+cFSKS z7M+kM!C*p6(kQng<+kV-JJ&3MGCzb66;cHjL|)Hx$9<3g>BsJ#nB&uke7b-p$q5u) zKoQj4EDGwe4xUnssrfDI`~Tf9Zarev5YBsI7A^1}vR0UeC=p2Ts*6JAX}8i$7^7RR z#x}w!p{4-0zTMyuUHZVxDYa9YlDUQ{ybB3rgB$;BARenwPWV*y{G#KI-!wYryN)+( zIu0I9Vv3x=L5*oB6nGie`i;jv>yl@8cefwCab0>zJKg`N^+%m^$tC>*L#vNEoEtZg zFbcL!PfwqF?$&MVw)1t1<%g|M0Qk4_iI|ylF1+A`6E}bEGoQWfrf+hdrEj2b>GG9p zk307HFS=~QrcIMGv*tADAZ{c!Ydgf3D^uO(TVZErrUsTSd-FTq^65|g-OXRQWoCSW z?zQ2#V@CMKC6|fXJ6`ciuU@lZ{WYKd$HyPu#_5u!E04Y66)!#e^erFm`|EpF-M4Jz zfcDYEi-wmiJLQ~bjgRyG`SHQQA!c+~m%PoFElxb;ByKq)xotjiBabWNaGuRS{=`j8 zoG2f2+=iUmpmJ`*aU0LQVE_`UlQ=_PLW&b8FVF>FMhsPx0jT zv6bb*tVzi*1Dkf@~;Q(N*XC4eq786tH zD`fRo>!F~jfsWmQ=PpUQ2gAqGrHhv=T{$%|?VDZV25H(?MIf-ack;l2y^n6&CeOBx zV~^dyX?o-o!jqjIWg~c3NBgqHJ&$bP^@VG$=cGYDFX$d#ddBG|9liGO10z$Dlk_Mr zZx|4>2l+%j?-RJ~JNJI~?(gz-)ss&;Vcn5Oa<-A~wC$0H@)%ji;j7nl@;-oOI;1H$ z?&1SagF~I)xb4nwe)D$Z+jQK<^R}LfED!$R5nipqsJDLoy18jhFgZQGYu9!~UetN? z`t=;7Vc6;Ei5-vc;GG(naKpw82RW6gK|IvivwOE6tKntMJ$;9-I-H9%otB=^1#TS=J^aAbG%qQeTYcCO zg98Ivl6Tb3%yOtWFxYv|ec!+5yZ7(iv-j{dt4=@dloiW{|MG)>{YP(k$FA+W&cEc@ zfAi@N?B6lUr)Nb%x6mX(n^`UEPihGZovGLbXKM%Z2pR&DZst`xjZGNTT0PB+#!}qd z0OLal=oRCQ2l{X&FJ0{G@mh8`%I2BnCuf28efXA7-MnLFewstabXpt?HzNQag|Ar_ zFUK9_7Kd%l19r~NTz>AFUw!4)9$p^!Bo_$X`uKx)k8J0Iw8ob)1y4B=4>C)Pr9{H; zF}cK~03<69&{?ut*g^(8hsls3P_r0g8RyJp)Rba2HjH!VDMQs}#%11zN60`5Vp6av zf!{I=8HwZ>F^OMpc=T=}E*rMBxnfr4=y2N@Zzt@ zk$ogfgKF~kR#|?m)nH)PXHi85s?yjLFy*>R1Gr527 z#N-H53RLPFTFM)My4v}C*8a)y1L);YPL3s4(v+7zkB{t|<7y8N&& z$N)oi>nK$J!=+pUWgWp@5)1xAH!P3llziWzeh1zG-i^&x$dt=fKB&o)$=;^xvOpOo zMvQi~1=EpYDxJO9hOZ{awbH&mIj z8t2@GykZGoFPvo>I@H~LVE@S8U3=d1)^~j2AHO_3I`WEN z`NiM-&%gc1qmS`iP~BaO0Q^_97(j60Qle(irxg&XHdHdHW~x$r$T^oE)7Uw4i&pV3`jJFerIY^UFY@sRaPERl$LZdd;KztRj zWVf-Fy{c>pT0V$9ZP1#50kmmKjb?^knE6$bkSBVlX7kSDSjq`AVn`xz(3|HoM0_TN z`0~!{i3uK{3J)VFvI!bSbIksosXJtQciN(f$=UHqZgueFUPh=$57xmsuCVkE^zedX zNcjDk$($$cXz%Uef(bJ4sy+usWmIZ-MC;h2zmGdGy{~b4lCR%RG05?aY6ua9nWN*}Fs1_3L#ms4_4Yp2$;s(Dw)Ps8Lpd+i zL7ois&QX|K7@wRln4TdigQWVf?d##SgIKVgLnL#np7R;9xmoTO?c$p+oQUP@3nwQg zMyE%{_rL9J@A|t>e9ZYEYybd207*naRCZ+F{#U*JweNW6n|5v=?cj4X_6HS)Ziexk zLyI>`HO!}7@)#??+D8#`yvnvjXBRgmdx45j%A%%`Nnv$^&Zd_ks`LQhMX?8$E$$u~ zEH6m)|5y}H?$0bg9t-^0N51~4n;zpE>gk)(#3OESknYmdiOa^xc*?xk)^XTC$JLh| z`#7Ux?UmJpTQ=_CLy5G9ooT4ZnnjQ%@(4E+%Ng${_ckf*QMx`N3BD zGK~nw1j1P_BBxi;NJnsP;l7hl4oR4pf|m+u0z;=_D=E3LG305JLHJvM3H?ZeFd|02 z2%IKmu=l0c)Cn*8I`@9UOZL2+$FUKzgu6_*8AH{`20q;hHy1VtfhC%($Z2J)LKovC zBsX6nXD;mG81AUX_+(G+@8W|GP-L&0O7G2u2U*KhM#}T%_1wi#JbW)9pBKeg+;-c+ zMHRZYZxZwN2y$#!7Y#ZcTS0*)tmhjWHIDfv!t+L;=0r0@e5dF9aQ~8<|Lz|K7B4yJ z!m|&I?8RB2!gU zv4*OZX&QLkk0m2v4Ns2wUMZmp$R=LmPK=D%>naLtpad)xmLzJF=swxWBk^a6ITN!; z9K&fL5?_RjG3x}MnaaDr&O3bL@rzey>Y(`A=$YlGWdU4=!SBBB@!xyTt=!hb&Ef3B z+#qnHk9BVL2oL`}cScX?9O`Vl=#14@J@3TTD~AIZpnut__&%_EuLm$pL&7lODk_|}<=kj`imdM3l8If+(ktryNh)Jc*^!peR%f>@InL4(U6>YeJQ3%VM& zlOTop5S{6~9A?3nrD7FG5`^;)Zfqhs7`ohDVs%9G8^oTvVF8PG1k>Nu%ctz@K9Y9I zu0->kI+@|&yZ)XYrubam;ibd$P{`)sN;zeUWJ$!4LMyZsv4V1eah=3`!{JA6en1PG zoYv2ib70Hi4T|{myj{B}>*#19B@9630S12LAM_}vU?=a`H~ab+3pp*O5SS&dg0sSq zIWR;Wl#)95kR`ZH17~V5a@?aSR3Gs=N%Y)WHat8uJ~hp0Ce9pj<9lyU?{{vw<&WR| z_T76%=cgxs|6PBu^}>reT;zM=p%AK>6QMg&YOcy5QcaN5KteSO%&OM7Nnj_QsYVL+ z!8X7sYtKevn#vS3z~$m&))uiRyMEFqCZ8;WCkLNNfJ3y7L)~# zi16hU%f!@-gADLF@L}5`8OD_zI@&#A8J~8Zk_ijM?5R?One&L!hmOI}yZ69k)f117 z(@vxhj*d<^4f17w1iP;)hYLzbuRF+N zGchEj@`PkOtv3YioJ5~!~{HH*=b&{uTZH!JY8J&JFOtX?T>bH-YR3d8j&E;(8VRS8(A&~ojef?Vw#bWYpsIqtGJ#yY_iA}h~k7(30b-8s<| ziU}Gf>Png+6WR>J5MknPA1|swH}WNE6(LH`2g{=)?GySo2*AYFkd4?f2Tuz+a&yK- zTd|YZe;Mv(G-U@&udE{d!n!Q6zaJzCLPvqR+cqzY$40;%dA=OH}L^p|oHk7{fVa{j|T)N&T$3I^87r*yEx_bLBe)*LfPd$0*>ce_^dk^g0|GCe7@GGDAf63Rw!vp8M@UkAx z`_4>($Ra@&4P|CfIC)WUHWL7)Z^6dWi3c>&?;t8mWAIWa=86rZs%8NZH-?X$JqItK z!6ct(;+8n1Lb=ph?PBx9et}_jn#JPZ2joOk?rdE>v;4Fy(6V}BCh-Gj!zfP4N-tGN z(WGbT!fhSP270f&XzlZ!y>YP51L=m&lNB2VCXDnqiHEf-!iu0J9;qrRB)w2-N!su; z9Yn|$1*4xT0+&!Dyt8K(sAk_B;M~}nCYDKH zbQW&k$lz;)N4W-sd*Y=dxb}>4A~TogfQGk=kz?~w7mf|-yI56pO6krD0Uxg+#13jo z;wT$W>y`iiSI zZa%JkV1U!ld@twgU%mOm@B6@>T_bbTQ^%ja`RuJ*_Z>Lkx4Yo95R_pG6{;xmN9Z)u zFJ&=b@lPuO;E@w!4Ye&-{T20cwSoe1h9M^>dBEyg=*}eGEvbLYb2Q!IVm)trBvQB$TV; zR^s8@fJvye$QoDpfttv$;1Pu+l0D`Fw&cqOZqPG5u_$QHlX6XYYVzIheAhpI_KRP; z;nuUBb^39eHVh07j*X4nf8T@GUUU79M|Z&LiYs4q<{76P7@1^#BQZOflBjB>8`ZZ? zbERV#KuLqZ)5t%t0^~m+Lf5`i5gQcV{A9)~XlPOlo~;H32aVP}v;0&o5PyfGC0!k- z#BXuD-qdMaH#pd~dHv8;&pq~8TaKcar}`<&Y7`RyVGl96CMFC3Gex)(P8HMyK6L?R zZHb6+49kuu-zX@HJ)pV)TO&_cMNMjHxKLp)bz%y*%zf~aB)~{+jST4BAzS!LYl|dp zg|49u4F!>iEg8M-0ts$nUC6YSf}jg8Epc5V9d#VI$VbM9dXOCCTMd8*1v_{>7C?qf z3SNZra!sffD7Ce#c4DHAwu)Oe<#GhGywieLw`n*D*<@Hv)BraST4ef_5>3z)$HeGP` zdT!_-h@P_G3j-7+5}Fmo1V$qUp@Tg3sZ6qARIwtQea8r<$mGBWZxc{yv6aP$G|ZGx z7Fk!JYN$ID;MI9y2hFkq&FLfT1x^mI(Ky9GF4axsZBvU0{mF`AnR&C8E4q?Vq{3;U zpJS(7=#UX`3LtfOxmHS+Mw6%z@~Iati`+^MQ>Rd3DTK6a4NVSR0_oH|c!GLXc|kD9 zikb8hxuq4|a9QXT9x4f!D(wIR$gs*eYJ~|?YyB^A3M^}AlSlx9Kz+Z4hX(Kb_H7^i zz+X>{&i>%OM~>Zm^qQmA^46rWk-gu%@f&-0?}y^b!Q>0aE6Tl8Q+VO64XH1PzF4mqY*B+ z0D%{th{Yln1pzZvvJC)qO#r32!0iMfDS((Rf>piZ@mI`jX!06B(Osh)5{#lp2!X;f zHpCIil3z>45SI}JPYY0(=LrG9xVe6Ga+(`72Zo04yX)TX-Fa{CKrhF4v(x#O?L1%0 z9(w)@pZ6=j_WH?j9-!wQa{G`JSV>ahYcADPG#*eRL|PD7k%ULaC}@jCbYLY_cloKK zuC{k2TvCskVWsz?cn1&Y{y0LH(V4P|*L?W#h36mjyo)v*eZ&%KnmBo? z5?2>zF6hk|xhtT8iZW-H?1qV)Ozh!dAT@E6Xd<{gkOoRI=^hE9#1IthJI%zDQW8wy z(wAT-i5w{a#vU_k8BnP(nY@yZl(4K2{S&mIOk`)oQNr}Q${Jj5F(DOBLoZ_!JrnMk z^){kq?i@LV_CJ{i3AngICN_`dl!}7cYakc8q-t*~ZFpE4?K6*881)gOdXt6~HntNI zxQHRFs*dC1f2s1fX_>S}erEs*si%Q6x439n4r{FoQDH;1&IFeL1?h-WSTZ+zx(OOz0xyANZ%ypU z8h8l`CNr8+cxLs}w}64H`Hhu?Qlg>!VbPc%+BuguchZI>uYJMsr=D;m&@G=ob%7g~ z8=3@zK~x1}rQ!~m5;X}Pr!z{^jCceOdkSxg6IDE`%m8fob3>IgQ~)fx^-t_V0i#(& zjwNhw{<2ZD28Iq1+U^F%26{8)R?n%X5>qZfET9J~j3YU8VG2OHK^zfmD~qRXk==S( z(tzv;NQ)CV6AML!QNPfpf--*=s>-#9=HB%81@aQC?g|XhA?j zkv54LNbGoPpj@n>9U|@=%GlX|lB=iJ0SqdkuhaW-i7}>$jNlxlVob_ukD}O47iy#! z#H~e%nEjnF`xmk;7$AXbh&WJTLN^7*5M%}I2@X9$LvTaAj(R-O!0Qqc(gPo_mxQhn zo_=!(lNMxTcuic&8&Lniq+0s6tBxwu8E|>z37%YpfTb&U47h**>uwRWMQIya(1aKj z95bj=s%aDlWEyH(K{tYRaSH{* z<7ZYsVGFc;T5RGLoQEso*tkmHaCw4n#lPx`%`duSBQG4&jq%uzA|3r(%5>1=-$X_j zQ%4`=GI>jiL==6+OCkx&^mh;qaw0}yB6Sjbx(R~Wkxgg$<{*!r*Vz*p%9TYOlO0N7 z#E9a^MT4a1i4yVY6mBk(wK8yLxe3r5A{;c-qlmE@{dt1S>hfdC*UI75wQ z#Y1skALdIVH9%px_R6kO@Xu|qWWV0wp0loc!Ku%`boaK$AHDy9y}S2NIk;r_h@;jY zxp95Za4+ux@}U~+ELwiy1U2m(;u&@kZP6gOX1=p=4d5t3)T;gwAXlv)Sx2fNDnn}kZK7vbDc(h=Nm1Ix)2;&Xy8V+^c~1TimGHU?aywM(`%8z z)Sdh!Skxjj#9GRP4q7M4PH9NIvIWph+F6xQ0IRa5Fhfh@P(!0TnDSL7WtBkoA{DZ! zsivRC@W{j09=qxI zRYx8%Ff=&LBbt@wz%K2ip)9d*L%oDT+zeQ@valD95lI7S9K?eR2P%v~)r@T@k&?|vv6xMiWpS0wO+dHWO&bE zTRdT-QrcW_?0Z{IW2k6Nn*bP10UJ4k6=Yk*Wyu|my6&0fCvAZRhhu!)lA93Hr|=KX zz%y|v2MK%*v$u2ZbyuJAoO6!h4HgJixIc=7hC#U0yD~T`Ft|c7Ya5ZnI5%<};i{v% zqr?Py32yM3ysBdo^J-%SQ+&Iwk1wD?o$v#z$>`0nqMo2S*_ECmfkBU`+)*N#4C+dh zw~Ph~en}|^mJA?I&+)?zOj_+!q20<7lA`z2Pz^v7L-a#a5-ZEH(ovRlb2}Y-AW$h0 zb>~lm1;r+-s7O^PE3-V9totax`bl0&83P2THH*gaiC&f(4yiRB1PKjEORE7-L&GyS z+O-e&9JzSy(Wh?W%i7e>%}mVBP4i_do&?mfO>zzurQxhW1AeJJcoM{4z@{H z3Jo)MpaccK`Ii&Ka#`{=08zr z$PiHWO{!8)4Y{E9;AI0ca=s(tG=o2KkUAnotqtY?B+$S($jJ?QfSsa-<^XumZ5%FK zxR9s82rgjcJ!8105Q(#bbEgqOG~kgeU62W-J>70erO@m;AN4d= z6k`(4jad;$IRa=}MhH_e7@<}&z(s-i&sdOr(YQ~0s8@u zLd`8deD%fa)_3zDw;$v34IjYsvNDcvFg1W!GXgYt3-}U&t3v0!2vGopsw>v#L? zsH3tNL|GYkps$QrC38efK>>0#Hj&K~J%2KLD$CRB00!O6(&&jXv1&>&Xq;Ws=Q<84 zzX5^zuC*C~hBdU9=5mO&0kgzLz=f<;z|~@B05|gqYs9TVffh+Z7cUS%tr3f(5d%6D zGM+RnGU_N5FV$kaF-q_#5v`XHpri@c)@$FZc!r@ z7sobHG!6yrX07-(IFKSmk%N|R5z)Cc`J>%85D1Ebyupn35B2p74h}d`^t-9En{EaTT7FvkUtbZ~VFgS6X|1hY)Vr!arN8kA9E5Fsl9dCF!Uwk{V4l{l% z;ur`HB;jxT5hrE@|8gWyS2^GXNT#+z(wJx^elm0_x@X)Ia5g&=4>h7P3X0D{`Up0} ztzZLvD66HrRZo?@4SLEWFL4#Xc%@`zjS)3dq21#n05jDVpQ_xA8CW$UTPciE+RF71 z3%)Z9Rgl$JYCu|HEclWXdUNEmN|vM*6GlnO{6k>rl-SL?H?)Y^v#}OhlsNQSCMTsJ z^oj_(5G-*S5}ob`26UTHOIfw6IY~R%%*O8S6rrIV=<4Ns25x-+{0tXcqL}Yt?f@H# zqKPLKV&Nd^mh`r3992QFMk9>}UM3bLf`)*zlwvHh8x(S?$_z;z_AuikSSAc2kRiQw zJ_VTCB&0r4gt}Rel5Cx#XyFJNDhM2=R2Cj|ks*s_d1m<;TL8xqxC)UdrGz8(Goi!V zvDz2C@_EO;Bv54GrojippHi5@Aj%4Kz(#Y4sW^BT9XOQvdxapL&TR(Z)q|xHETco*s3h(*sxDwLO0cqn z@p1vG*MhC-R7?)2az?;xN|j-0drknzRxE0*im8-HC~Juk9jP#sXla4BoFP<51-)h< zfFVl@Qq*UbpST4KB>h#MwAl0xDu|Q$_T!Hnc=Zc6pLXIJV!0gpW05JLK_^LACH4pn zBHg&DIuE@a2AB$v81htukz}k@7YJ5GoFQmdqnTeKRw{NTt7-xu089emlsPTV3Nqf& zP10=KI(A6RRF_iXot3@R7!6fN5DSK|0Dy}^pMkH8-A_k_9J<)iVySl5t|9B(M;n)e+l9?eNRj}lR@8*EoC&7gy_l zS#M8ehc>nx19e9VedCNR2DaS=0CYiOrdEDeK`ca|QI>i{%ECr3?L$SKt}F`J1f%le z6ck1}Dnl~{DwQHC)l!01QeRj|9Q4>bZO^QJh8EBb+g8+MeASs zoa1?GPO`|4MM|TqIt%1H17e@GA6YQwT(K6$UeLs?L1r)kYeXSo;#|pvZiS!W#*T4I z#O16!gw(M*i!@0Be&{3t?+jIN7JH9#vy{Rkoi0*Q$xO*~7g7pP*{3kwRRR?#%!o+` zAs2xGJQNVv=~k?r&SoY>kQm&GBH}s=u<{K_uqg8nxhAK_dQf1&j4aYNNn$h?xw8+B z)`)GBiHm9RvYb;G5mF1pyrxYp=%RyoWfBDqpM7wn72t(ngV5~Zm*~@ADv~6f)p_&? zVWmq())K)Hh<8cz&6%Pvq#UA^d;vfKrk2qnY-Op8N>IYUR-gzR!IIMsfeEjIp#~$3 z$v+E~`Ii_==!R3pXXsk{6ql#h0Sxx^4^QNxIog;0_V75(RZNutDxJ$_EgO`b^gxV) zJM}7NN`-Ef5>9Ra0i(H2VA3I|)l+&x0B9se!`C{gX0QUvAk6QI$ zW3}al0iswuv;L`BK>zAkVuppy?CiOx9R3SeoN(;A6;>y5evHx>^~Q!v<##W88V$yZ zoxAB{R`5#~QM63aZ6}?sDM8*U_1ZT=Fl zZ0sMhNKeU~-X3!RsgTJm(i9#nVxFSi&_%K=5dtJ9+EiA?ip+kuAeHHE0N4)82v|J} zA}&Qy;|LPzWR5=k!!yGwEhboDl`vRYr=}3awvR%>UzC8m5Sbelzhnt4Tjw!ksHB`Q zqNJE0LPOC&qynQ+iQb53b;P7Kr-W$|Qod~mJ3ZqVKUy1U0j7c0*vbOXd?h1mMXf?Z zA;vX#*$@RZfM#*8`FZkNI>3agZMm@!D5aN$D1M|k_KN&EOFB*jEs&ohq%}=qlx^c6 z0iRiZmKMlE1Kxkxe_8cH0f;YdM*_>1Jv;47$$uu@oh`VlGo~H zLKA{Mst39Wr*)Tm0MjjsoAm8FBo>G3mMAsDfmNMWTp;PKxt_jJSeQm?XUU$Jkx?Wu z8N@5pE2h;TU0=vf&D2L=S$hV;TZ&Fnq1{M0Of3;8EfTszp{+qS{3MdzM* zF~pbsWw7>?2DtP{em9DCNXBzF{&D`8STQ;sI&h?1|vFAQ+rM*Z8F%R2zy3a z5b2IkbH`DcVIy;nf^W!)7+?$1;4?{ya|M!B5#l~l@3U%l>-0f}Ry%{UluVlC4YI0a zOT9^}AtZXSLgsKqG4xVeS=(>jm3{1@=+c6M>IcAfx}?vnux;sSCREy*K2)tlKw`%d z#TbH}2G2&@)7T;;oG6M6B6|>4!qf#^N?_R*vaXmdxeFu(+-V_4qq)Wrg+#!!1t7&p zCRxBmK#0P0(gd|Ucl(bS|Alh05qiHYh=)faugWP3Yhc> z73lD9Dr>wE31t8XZ9+7>M0#fV*;t_Qh92OjfALDk!9~wHam6cMaQyKbR{i*Wlm2K( zC9DW^f`=WV3D&YpudbeGcq^A;8ZFHxL6qo8*3fiTZT{18-@(pHt zS!t~hEcNt7;<_POIWy!lV<^I{e7hi5QWV6YXP!y7X^AAmlb-?KIp#SRc<7i|{Og%qbIj|dL6(REG_xVcx7+Y z$Az2`#Il-={~W@iy%=$R;Thljm#Jur;WDt-NGy~g(;7+{Pz5S;hW3I8jeCn4gwVMJi?wkO=Vz5iBvOUf@ubQ8uEQS5zQi4X)CdN00}hHD~_Z2d*)c zh0I8lITQp+J52~}nUzx3PZog@+7@;Y+buipHTDtnndPTpfkZwxPnrmENgiu%U)kSr z<)z0w=e!L|eBoGJ`xB6b(;J4dZ!k6thygFtr(`BFA`A)t5?^#4Pxl+HO|NWqZ*eLp z!LpAVPZexni_vuRChlyc(y>=!*$F&!|3CKb?RURy%j=VD&#y}K!YZh}f2sPe@AF;H zoMVpj9AnJ6)^m!d$b+kP6b!i-=S|^6<6v(#5Q+NvPLZyK+lY7TyOmOTk z77_I3v96i?-8Q4Sh^xA^`m~&5pnW!F6Uguwq-+2dUS1lV`b5Pr{G>n51)C@H$4DoC zMl=#?qc82l3}>Hh&ni5qt?KDbk<@R2EBkMJoH=Wk38nS2u{WG*O4^!kZDvy zM2&d58l>gWh}it9o;7F&ch1^P6IZ4)_5|maevn}=pi?D2E%plubvVr0!_1oAMK)hf zBYZnZCbAi*vHgLY(VH@(s2|iLcsu_FJkazsMbDPSG{5`wm%sc^{)4~xNB`ij|AjyJ z-3#>ZLoLxK4uwfA0ofs7Za<(Seg<=L?7{#Pa;KgOr}9}`T^Ea9qTKtp8&eE7y8TH= zUo&j*VRKT%s7%^TgcsV0I>ki&SQ37!omy#h(NEkg;!}kz8^+Bv0?-OuwYQ zEU9D0hVfGUK|e;+p=N`52}*2lQkc~N)`X||%^H7=s`HB_q_lUD{jYF*WEU{zBfG47 zO@o7L7X4>AzcrPMvkx&nC(R9}e+;-LTK+1W7ZxTor^&jx5gvaZMfl21!&nH_0Gp1m zEzPh6@T89CrfK4Cq^r&)^N>z~uuWKL;3-BkXrH+`W;XQcm*7}#R4CMNuweSv765&G zws%R$XzF!76S9W~P;{Kgr(fr?`Kk4X+hxM?DYU~1HmQ?4jljJV zBM9GNPq(@`mLuGHmAIpOh5Np(cy0c#*A2a$ehnTlbB^Yx_X{^C z@BYG{{>eZ2M}O;Y|Fu8(iQk32(SQAlG~uf5qUB;ixGW%3q!xXOkwU57j%DG3M+66N zprEnY#YZ~1Ezwiv2xbVR96e+~_pBYo=LCsMk-SV9`g)7-XGq+pop*QOP9nB*nvfOl zdeZ~H7M|tMFhN&BA0fV{u-}qReXQV^U))0Ix8= z-~WvdeIRU{5$I!1&O?|yrLsdXdHHU|={>z-g&!+k zVuEJ}#rBBtEdY#cF6S@L_hE=R2K1<++*Gcp-n89hogS?UlzQRo zfUH>7vH`1z4Pyg5Z>c*psNe1QxGrEa?MyOpCd03TnzzT%Ip`eCsoZkD@LquVe{(Q@ zKc0lJQ@F&{2WZkExw&-ch`zU%h4Y%D43cBaH)mYRMIrd4X% z7jo|BZ3h-rj0vAD@@0fEnMNNzTJmT(T~F|X5PJKNDIf~Vx1s}&1x@()%mP{$V)aYh z^WH#CV{-3Fc%sKv#lpE8sF>gqf33Ouoa57G`)E=hf=vi4I5oLn{OaHS^FRA1|LEWSTYvQr zi>8Ia_oswVd2)6kT=u6*A079FE&{FK`S3b_Th7=1TaH*UE4IK4q)2@S<<{D$} znFc6G*@W)RdU#z8XIQh_LPi}%Ws$Sbeg z!-&SdyP}bHR!a-miQFz3KJczIM!xL=KFbi)rI;6a;=WoUkXP2O0^uBoAXB|li2M6= zAI>2X3KPPk6`|1SYY%NWebqqeHIwXN>V!OlfNNa>hMlqJuIRh3G<(&D%127lX9`U= zUy!g<;`GTO6U(UX%7k{%LQE)LzL-mEh11&@DT_T?g6>l&j z^ru_gF|$|R3{tjX$6_^Q%ln4Xa$~1uF(|EA*(JL=H4x_*WMNu{+^uob^6)bS@mmG&nc#Zv!7MFn zPhPZ%HJ!X9PA&6R0&5|8%n{JncXjK1%iMt}LqU;m3g`yc=4Z~u+I{BI;v zsD01J1(x`I*4YY6KO@-j#rt=CMV`13ze}Xjf}(P$(vLJeMP}bbS=LA)4}Uh+g;mYn z1e2k^^5NIsFch?`zdinH&sL(SpQQtTp_$x{=sxL7!z?+8SOQN#%1}$7%dw{m+v?k6 z85%E~P-E$s{%J{SI$bffWMfn{@_Y5nd(a1uR~CR|{xm0?6cuBXdmd8Bh1H~Pkb9@& zsXCz#a>pNI! z%NZb3Vz-h>t^%0ZVYUZ#y;(W+;)Zu&R0*P*Ta!7)vnm{uIELIrjD1PWA8zTicfQ?g zDVzjlC1!3|Dcl6nhgQl?jbk=gpdf_AHuA7y;AwXB0@@2L%(vq=;{j3f#UFq6`A>f5 z^Y{PskN(Dg^mqPEv(_wr--jG89KMh=#0jekPT|q1%vvBVfC3e1JJnp+f*(~;>TA3k zHnxC=TncVYF8(BAN9F3)Z|oY|SBEbW@ovQzs*Nobj)D?l0GF}tr*Bane)+N*i2v#& z8QTZhaunGqG%3<UrTr zCn$-y5kB;xl%Unc0Pac*e0p*vO@>W{qg_KG3|H|bWj8Exg{l?!1jjZ9a+7r-o;8#8 z#h06xN4cp(C8KpF@u{^j${Rex%2lwtR+)8F4%@3tsl=!N&1JxTr=$4_hU_%F3Krmm z%<$2w(pPbgfVb!p9&*)dd(i~s#tvh2ZNSfHLj11B$8`a-$b8S={h8Xwg$!JX@tq}h zLE&kRy2)vSeT%_@;pQ^m;TBciDlX>Gam9&eUtSJh_wIGgo8m>tzh>-QOs@q?lmd-j zw0x~u-GtbLbWR;i1wtgNAR|6T%k_oF=@|CrbWG2_IU!=}8yP!VA@??pht;*y=;#(* z(^xEa7ldM|n2d17!=?m}^9y@~IGBKfWBs$-M)#OCw*8Du(I!-<2z*7_TQs=wY^iHB zb<=ufDMmUHm4IRH6k>e@Q)L_=nD?>hgh~N`;~2et5=vGCi@Om+d$r``vMB9Bz_Q73 zp)~Zdc^QoC6yz{m^3Q;C*;s5Qr@y;VZEQk(JAO?bSe@@%Yd`sq|L!0B*Z;-e{KMb> zodiGi`1@6%kd@2@gHXHjy-@4wMLKyOL+grMT?h_u*o#u#mY-Uyjz{22L8PK-JwhO+oh2zDzSbino0h*L}d#<06=ifrg8YW8Hg z{j5#-U1wqHdMSNQ6gbhARo@MRG%qbAoV?YtAW%$SJRvunZXS_OjzS1 zXYrgv#B4uQ+JunUJ{1$r72XL#Lf{jsSDYCCcKk*?&>IDNRp8J3>{tKOfA{bG?Z5u# z%;xt!CqxvZZFyzPrM55?RKiWN$)ui3Y6I49`-E3|)mb1$qOc5qYE#`>Tx#5AL4eL4 zrzi3Y(ip|xedg^eVzAhT)F_cd*bmNKVHBzj_b3q#ZOOM1@ls6WZ%lZvd4P7_x4pt) zhGO5lG|Q`98U?5J;{-?>x3`(9$87CxPheNM#ViBDi8_OMbEGdrb#K#uehzO%9=t~q z?O}*wSFJU}8s8MmcZ0(5hMk`H$x|xmLW(R`D#x3;ibwnP+U|_ULW`~&AIUtK6s!PD zQF;QLRa-2k8z2siT+8!=7}uyD-C5wcy%lCVikVx$*t`%*_*SA5M^EyP)tOvqPVDhe z_dH%oAK!0z{>Uz1ZY`WgwoJ^#4~~4zZ~!j2m4w0h>SoEL)s#;MTZ4<|Ot@xd3xS~_ z54gx#1Wg=FOE@_(@41#)H4}k4>5T`JLMh;IE~7+NNI{>poZLWB<3^?9_*4XHrw+K~ zS=Zt=In!gUB}d|(jG`8bw4%eDFbPoTG*=8^;{i}Uc#{%*LlbOduaKtOQkRlv7`D`T zR^*8dBGHXJ$Fq139&NT0S2xagiT1${-i?#u+=RoFpBbG<-9InJL!CXRfXWLLlPM_H zV%A=I7Ht_`RR~ zXMgt({;PlRH~Y{_6I0lH@3IM4)VcH(4~vZ&BIUFSelfT>qtg?IuqCLS<}&ydd6r;1|R`k{CrR%Ldl1lH0M;Ojhv6C@_H0Z*B@>jYV} zQe!DP!y2-8w`Jc?-gQY3KKsb>n{k|9KZ}f{Oom)51xSlUHBY--M3^)tcaycBuzK)= zq;@0SVr{QOEcIa~9$X@Vj72pkF}cFvxOwt3l;O{nHR6yrcHj;k0BhZ+QR{~kI9{@c|aDln)2nU%IGY5!o zJ^ZdW4er&Rakq{Lf@u|Yh?bkRnBK1;5d$8MNMZQ;JHMTN10Lv234ir3{fYnnAOD?y z=P&;0CrZA5EjH_mA~CR-djds#SlM)o5oudEZ7VFDg9A32t{Oc(F(s<5hKwpo!>I2l zcwG*`>OfvPNIMO3?r=l%O6sTykwi!)H}wpt0I&P>^niEDuASV2&ZS1A6giGZ<~uKM zK9bXXL&<44gr1YQW&_mT)7Q(&VKubs_fi%)Ij7Qao^IZY%c~F)cJ9y`HPRYoamsj& z#rsgaKt}u&g6Lk->BD=qnk_fA6sjJPN1${(R)SnJQ1zg)S2rI1c5p3!o}fclI3AS7 z##rAV@nFEKXKMf%z%%_j9#n;p8+-ri`}8sdX=?IS?OUn0;~d7Ttf%hKBJtw%g&)@6 z{`j~qU?h*%%B-mK6@#w9HIM(YofDgHbCdUt&2-bSK;a&A%SJ4C`WPL)qE!bhOB?8HVOE_RRh(L_Yxbj+vDVj`Z8zBifBPNGOXC7vXi&eO%d z{3Xe4(i2PJLT4RJ$0n|xcRJap#$ru#Y8hF=ah5XBiWWd{?(HcBp<=&2}-k3w( zRH;Fe-zfz*g?yhLr%vN6IOS?=uMxsyi!o9rQXArs)xl%_h&-+4*nOhzAsqWK?S}}C zDh5{@^At&gVV*TcH62T^^@fuEjSivhOr2LYb26Sr-;VF!15IP!3iW%x_p|@uAO4xY z|9Af_S$O{c{zWI@X8ZD{MBD08R6Kbj3arMdBp&vE@2|Xfe}QZ)lI=|`TD+l!1{`m4vVlId;g0ZvE&#Y;#xqs`l za6JG}QpsA1ooHK;nmMtoL^MHG1IB{lTC7r~mE0{}=x7Pcron48RAw+ z83f~VRSab=UXwNMkVZ?Dj_a=zX2OnkrKk0xV`DpUlzxnkqq;Hxyw|~rSG>$G^&chR zPC91f((kl8cqJtxSnOC~WUFN>5>2ur+W0g(R7J2jeD}%cSM#THm@h>0_|Sa&r;qCb zrWw6k0$etK?kOjUjMr>~HB~&u3!CtB`R4m^=VFy-q2FXb%g=)w-Z(V6dE$LjKF@M4 zN(I1TU@FIPtc23!n*WEL;lQU=*mb5I*hJ{c1N#{<_J|ck#|ub{s0m~=MqUL(@Dh9N zF=)IYVpA;2@uE5*FpMs)&$4w^keGupRR`}^L2IIM?qzi^4k1b*26`tPy zH+q(C_$6(zKiTL)fb&vYZ|z6@*b*~D61c8eIDA6Q0`bAghM-lM=&25ELK=$RNDMVN zBe?`I+8cyL4ba%wyD(|P8=3_TcI@x!_s^ZIx=nx=1_4F0$@6hOlvCaykoL@%ns8`a&;rf^`paEC$+232hPjb&#za z(5;h;#Jycggl`dQJMHe>>Pl1-S<)j5rLGNmvTF4b3KN{8=-GRyI`1?7U9!MoxWDcS#Zt-5VL(r%BnK8|iwe>hbziEA()ZszfKh5zcukHUv{j01c0j0y`*c zRyl!KA|~*L-_E}Y5B&Uhe(QJs#7~p_q|&cV2lwf{SSscgAR0od@bQ{z-G#^YS^`pt z!gLm~7ek9VgiP=x`}Xcf@!P*m`D2?D6TKo3dc!sFIhM<&-goP5UVJs=>$b4 z(Z)N%vjcB>4DCVorlut|7Ub^8w#%{gtmAFsh@cXolZS%x$NjiYE3N^KTBDQ7ffo+y z0cgaYVM#W*cgX9`lW_K@#0K8Z>%m7Cd6Jxy+8V1d#{S4Cz z!siw9$%^gSd$y#><48g#0VJjtidh6~-lkA4V=4oPBE7?lW~mb-*bc9unAfpbV{tMJ zDD&3b;4WZA3b&eg&9mzDzryixUBH}2S;m*S>UC}QM5LIJoBjX*KmbWZK~#WPlo{Og z48ogx2&K8MQXh}zVy-i*kC*Z8Va?d=a3m`8Xv&LFDD?AZd1*#NmrC)Co#0*I~SxAX}_AcsI?^CH;vl6FUg1z-=+CvHe0bo{ZoqaTcBn zYlKSFQVnz(_Qao*U|Ko|IL8GKog)N5LVl)92P0F-RgD%Fs~TbL55nm2K;QpEds4pk z9tVyr71I1-duop;!CSnCLIE?3_ZZg*G|kyZtxomV?i@Bz0Sv8Gf8*7TA%gP+F&z&v z+(?3@UA#BxjB;mqE`uBf&Zckj!QPJV*#k|~@Bd!kI{H0}`Tu5r`S48pu0{$GnIV-b z<@=%Z-VAW@(39_(C}OBvoVb&ZanIuPqO2#eSDf`uCkgfS0ZdpFB2A+A5R*|j_2EEC z6EIm0k!1tj7 zw3A;VY(Er5cxa4Gr#(rg8V4hK>D`QSi`6{8bi7X$Wnp-yxDO9duM?x}6>@f;KSnm9 z4ThYp4e!7rOS=f!dnp=gB$(Pic~p2E!8?XyzShMqMS2P^$n^&s+X%4OmMWYv>k_+b zrc_#;8lN!k!Zq*V7)zD)>kjoQi0v%hP#PQ?YJRn!{K`L-s%d3Y5r_1iIw2%C{zj`( zO88iigngZTam&3~ZWa$`SSRJQK6Re<>{m$8zcg*0xQPD2v(k#N9A|aS1opSpVT1tY z-Q+T6yv~Xl7saXv3GVvbjE_Z9tSva8dbCaA7y^+up`6p?;`KPEp)dEkbAnG$oNKJ4 zk+NBu(7hH{H&Mkye7%L(N3EV&BK=z)AK3-WE4*4SE!UYULbzZv5hOR>zTn(4FoX_s zR;%Hh10*ndWGNAEo!OKDqyzM%^}GJ#u|M_LSq7yX=0NK}e}HRMUr zCt>djV|iqqN=k7~IUMw90_Z{oy*Q^^x9{3DlE-C~$F9VV|s5>`{1 zvebi1-)U)}+24mp?-k!Ja(eM8)L%|g4FeA(|pOrl%VbV;MH=mTNAfJ`qh_Te5P4PB+j33u5 z6E9u0ccB;t>MFe*0%87bH+JnLBz!}WTF z!hPUPknhKVt-3wjmX`zgT@JyxJ%D#*o*mI+86Ly1m?aOuO!}1LsEqx^o7vdK61uy!GNJA#<6pLULeAiS1i5N7u+i*P6$dwrv(xI%pLKRHHvRsisC; zQLBS%34DerX#?1pJnC+Kt_WZ=^ka!?Mo>mWHEiXnAopt*=|5*hqwVd-SJ_smNojzGuLk;sB+h6rf&FP8Ld2ItpO$5{fn#Lqtwb;zg&niv+7vJUu}HGGnF5OiL&<1*@hl}431fh=S?_Ye zVmlg9Ixf1*jI4XQ8)gL7_pdhvVlHW=z>+Mu)qdmC@v&#S`H7*#NgG}S+)FXxRJ+E- z-h7c1uk*-<$dc02tp6Y)6@WMbrAs;3-1JaIGM_JhXmy(pI%98-P}NzzLX-}y*c3N! zCW*!koiTHb$pFQUr%Z$4uv3ON8UpI*RDZ&Y^-HBluM|m_ZQPec4A=$0z8jR7*n**wWYtNjO;ChJF7rvU zu=(askl~j#bu&Gz#uXe=M)lQKUZLLYfw^r9KZX6}3WWWKb47z*Z0d5+Ye@X*7Rwo2 zGsMKD)wBJG3#`8T;XW7B(qgL>7bcLy;URUHkND$o-4o?Iow|_$b(V&dKFK;mh8i6| zw#3-S0rALF*isT3YR#s_zWwaPq&vyny< zN0dvLuksam0}OvVerz5P`BFq~Ed<1rg!xdciuQ)($TIQ5BoqW*P<6m5g0`eul*h{Q ztKRGt1|lS%4Ta>xv{ST;y+=4C5JrQ`=hdYGPayz{E%^M+n(1G~| zPd}zyIji1PI-0($jY_qo&{T{Q3$|YHn^ne$8qJN7#Lh^k+hu_$W9VmiGm&pY7F|?h z>sE@YrQ0jnXEen|6+gw=oVA)vOahJ1Z5Vu)BjS(f;GV5p0)Sz>mUrA{dcLELwzM&0XSN`Hz)>#!5Evcj!h5rJ8l|p4C*)& z=d!>y#6?&$hM*vzivLa4(2Y-z1tvv`AKzmWIGW?!v(BRrt#FFJAP0YV3<&^%QH^+&hmYYvTnzqMwf;v_YP;Y=sN3-Avzu{f5Bn79=-HTxRbnS&TH_J!;VNw~7 z4P$0d2y~#vdAe`+QNSncQ6qI{4a~RW2ke1jq1aGsSJ2($t10}9bQVWqMFhh6%I`BR zbSwauMB$++J5VjY>9#dM64j2PX9R65 zEJD)=x5>S*USWJ?5N@?Q8bmM!@Oog!qyF6loh63KgIVErP%LV<`ap|do;eJ9g=)03 z<4AF^xJu6l9)fQpP>3df)iI3Y8YtxHZmrjpmb7X&qd??8f-uCh|GFdYsoSP%zSio^qNgS;iM%cfb&t0l%^8jl=l)p)C$N$F1ADnup1$9>Kb&tmyZ>JxZ2Nr>1 zrW}>M9nUrS1x6!*`d?xmOg=4LmV-r-c)P%rXnQQn4oUP~0R(h(Ue;(7sC|p^F@Yya zJ%x6KqE0(9+t~qwZ=;kPcyTBzVd{Gn_+?(@FLb4#Ay`lPKdkA0H3yN(My?zD-Ax&Z z6LRAO)wM(LRN_QFnw4xtSvBB97KPB$kwVY4svKB7=YDd1sf)=pY@m_hOp0Q$b8O`}T01<*0l_ys4Ig_MF_n)h|C2&!blQnL z7Gp)dNB7m9s6gU9MxCjhpuNQa=MgX}vbA3X8BVlGny5o~_u2b<1?7=4ly7(XxL$$D zBUVQ~Z5?NcPUjrebmXjAw?NvEa5PsO+21a{H!D0fFK^d6z1BM5Zq9BmAyJ;k)&jvs*s7G)P|LSVsb z?u#QaI}a@CWKn-*hnSNivd`vNTW|_4F=z|l^Y+T6(^keqYR&%Y%l6sH5pJ-0+DkfX zc%|jyUiU&7Ul}N7eFuL5JhfaMCA$l%l7XJb3)jo6K`6Dc)sV3qBX(T{gr-L;f_iVj z(g)dPN!gbYXWNJyV&*M%sgt72k59u}W>?wAb|(b=c_|U@9>O7SmWU;Dc?beo*|qSd{*dvyIRxgEps(cEoI)u({tPtUA+}l(*wY z=K;adad~)wvXCmUiX?y~)%iu_i_%D*utSHsD2|`rEtV2f}0EAT>SqAE@U1esnK=$HpMt_|fW_`<*>UA@fh}C}Z{UX=lk257GILuQ&kwMVz?Td_R7TSQBTq=FX<6|py zOt*f_+aG+(*Jfce*&1Ebd}BK<#;f zP1YuGp=p#VljGU(pw}S4)2S`;Ci>BKh~yF&BKrP!7{j<(1v8u(<7Rvoix?fFcf~$k zUR)w%kT3uW3Hq^%`SG8?jVol&;MVD5Zm9p0Ght+K-SbTR)R^;9a^f@ed`1dlJdp5D z411U*g!^xpAl5>g0wY`Z=z~A#vkS#@M3Rva*AJy~K-vbU1^?5atfJRTZaz89gOoA1 z!JE@+``dIyH9=fhlt z-e3VN-*j2ERF!WfIHme+9WU5p0|Ge*%CuNS4AyMs^}ErA!tp-bc-VTN6w z6xat?I)eqeY*H|%F!3$&3Eid#0ZE;)V$?B|MPnF36~NHAC+xNL0Ib#=CO$ABT^l|Q zdpdijK^22$38gmzn7lMk)!yI+!14@3W+pEUf+@vc8_nYaFqVW?-{9oofLI*uAxyDG z>opvA!bxUS&rE(;)!i4p*cxqc#VRa>{cXxdZrZ%Qjc_BR%tcT2ArB~oZ!~ph`f)=X zkKHdy^bj+CrX}IZ@zK-vSi$-Zhq?QB4o+a^CPVa_la3s;r**Ei=x>y>cPv0MNA_%D z9!FcaF?GRrN2?z(TUH)e0&hqB>;yPU3z3 z#+1qZ=H_vOZ>Jxu2MUeSaG_^CSXj7+RCCFg;YFpmw!7`z=S&38MX!zXKDpPN~ffJS<9C(!MT#XLEYD-<2vd(NV4C!%F{L92} z$~Fl%MQI4@n4h#Qx70ac^a@iqhFw+QqK2{IwgR)hZgpr1bULIDPYJN+@DeSl*rwC= zF1809aToq~IdFNxG9YXyh^mH)$L!<0>&0bJIFrrSv5ojW?;1pouq z4FSoX7^iuA-N!*e+Uc;bIVIz!*Nxq9v4L(7B=Sd|YgNXYzl|giM;#o{31ie3ysZ(klvBEg$xDYi+xO z7OsYSJTsPaa-|?JYce38a`X8elQ={!Q2J1Z?l2GXVQ%JnIOHY8DidDErl4MbQ>U4l zRwN0=9g|aM&k-7HnTuwkgO7$?_^{O1x03=gli8(>mrbb(HL}A;NlF*W}prd3`fj?lQTDIIZXkw}*6j#Y8a2Lv`tMWb!j@D;W{x zVjnl4%em0x)YS2XfO|!=4y`9Bcb)QeFFT2OJAUXMC}@kaq zKug$wI~FIOzWbuL7ep;qx5X8;t|*q3Zlnl8-{b=rbfNDQ_eK*}M=Fd-876P`tt4an z1Y`F=Q>tX#;K;3EDMUBw8d~P`6nKz0V7)h*maHid?!Hcd9wC+^t2OkDQ~!-RATGQ1 z`N271j_J)3+mfK4j7!0Nj~cQLw)o^vM@)Bk4TM^+en8&foK7B%UK)GJ<-Cz%-YW`s z$W-hxmgh@Fl+)?G!Phq*z=>!(q|EqgpYc8NM#9b|5rxD%(;*H_-jk_o>{X`vR<;Kx z3J26`o6nxbbk6vtGE**@EpUkJ^n3l&Tln;G8O?`>?Zi1DFfjctbFoje7~ zTM)q=KySy7)C0nxAcnV|a{7>rIoGq?>qN1n)TrHXEwK)+q0MHjIH2 zv%OL&lTdc;uimyIV|26)gfzSWQ}Q{}zW9_xZ%nRz?T0Ip-qwqr z2hQGza+H3X$-XRPFB~VCA#_m`EdgQQ>Vz_TQ8=PcvoI9@)B73mW;x$Fx8p`OT_oSz z4g;>t9wX{cJId3Mhgcfh3oS-ZlM#HE(?|Cim`w7?&iUh}T_`a>t7mP`wFY{=0INlK zRUpV?i6R~BD@KQ*P>!(AJOT9|aN`u9chmaC!$C9>yiUCe=<+t@nXxU;453!AzU<Dv_N7jznwp~_2ZhB9rCZ5lQb;O+Q~W3%|X_3p|-L*t&HH0dZm z`#7shi{Mg9$yi+hS(qwWePd~t;uS7m2WHU5;6@N z#d*HPC<#+wN<@_ASGaJPBJHJ=WNMk8Lq~SxM)xLXgjea4%hUAVhHR+#wcKP<-{arp zPV+fb(d=MsJXSEdfaj7iXv7V+IP^8C4F!$&MGKU-<45KJ5$1ZhHpcJJA)1m{y zLNZ*S3mdoa-0#g_;<>W0+|>;xu#V1?AcR>YE~DfLLdm@Z|J5&l^~vXf5-`&ER*TAi z8j*>~IGj%9x2O%WL|-O`Q6`2(`UtModB{kRR%kPt3om-oKjo{0**uv z&UON8MfQ|nDkcgZZ7MElew8o{T@J*hzXmH|5od2#5%w3-jz*I*JA9Ymm9E5v%jEv> zwQG7>qsWk{%}2+>SLlDyoD3W;T1Os8y)=W?sYgS!!Ki;q_ta*mCVj;K;Q_;NJ%ruf zg%@2T6V8334|kHT!*hIvbL&H#U_Um50!~QW`HcYUlYx^vMbq0h6(Wgv=EI!6%8&JT zIzFxo7{;-Se;?2AUEp&e)3bQ);?eD;4;M{%pP4eqn-fE$#6h^+-=@Os2ZJxv>sds3 z)2%GiRs5$)yMuiLCSEZcS$ujfAwyfNh^=kfo2Ia6D4k+?!GV#M@DqB?^nP)fLZ;II z?EQw>O2A`dWL9>(hQs{v(eIx+X(A8`n`iBv0|UsnBZNE0(e0ctmUZSR&f$VNWYQ)_ z$pb|_wGl4{fn%;pDr7kUBy$=S+_YRRX~KXubK4SyJwHqw+i_;%6p{ejSHXICdWs4L zJrSf%(7u9IHors-YZ%@N_OPU$3ev6!pL1v3+IJq`_&w>*5Ys{Ygw!8$THU7Q6h-!S z{IET+(orDTSQl?1cHt)^ill{5QF2R6UwG5?LQ7rYV|)Qy#I%L=a^Hni3v$uwir`}T zlC%_gZb&_Zm^{tJ@z(4FE?eRdbKwe*{PdbKC(`b?ERlhCt5RkU6kdCju78#Uf*P0D zFw27cx}uWw{o|VzlRBJk=}8O)0Q-n)wHT&t8cyf<^oQq>b{Qp8y8{$@8mX?0DQYQD z1!<>6J@~VDgE)*!QW)b3YTxKgogQ{3lOHF#r_vVnT{>l}txGN%!&64~q75JKluV&_ z@4Vx05)4IiUcF7z*Qi?*GHY|kbUbDwMq|6`k;$;sR;Gxxf)Yl5!(R9ZMm>7Sve3K2$uG_wBQ_f{ARg8grM^ z#Ek$g0g4ac@_ z*w7|o7E#0~iF=}=?OR1;o5991@LaogqIiDZMs5~5KVD&@Cb@yF7;&qkM?Dln z+0$0EB;sKz&E4ZiQ$V2+C`>G=m`yj!&1NkZC9wkaNL1{`dBRNY4EIi#ezBLhP4$%d zwx{n-`E4cr|G_um`O;mhJXMpgIE^R!Z`yO`LbhJvZ(_k3b=B<2RBdXwC;IRWZ;G(2 z7owG8%qJ$;A;k_CIM(A@xSravDc;bSbzJG$R{i z_Vm7l7#4%bI)${>oSjoEtr5*nmSCVh| zbs_*KEhjm9JGRLoE6l)k!$rApz-po&eGCHNkuW$lVTi)7=M*%&QH>v-Z^sYW1MNY( zjjzTCp4X&b^jb?^F=(_ZZ{z4C1)(cAm&J=EQ54R?aY4Aq#|UQuUBR#HFLD!D-ihZ+ zKefnfAvm7I3s2bX{kt(?7g2yg%=slV({;FHI}XZl(M}GSqqneOjM;h z$y_i$vkwDUBBtq2KVKXDcKj$kAcqB#A1)M`g=6O@eyR!L-|d9g7IRJ#$o&h%1(VIC zE=ng*i6|2B8gzjz-|_6gd>L`yG*x)2MaOd~De>;IK0Z(GEVXsvjn8DpY6s?oVSmCk z9UE>abygYD1F-WI&N{IC*-R^SJpfZOyb_4WU>}Zew1cMkp&D1*`Qxw&kBP0?+?%%B zLGjY7BE68O#Az}?)8NG{f_SO;^XX`6QgZ)~$|*L)WAL~xAUKU|`5q21JDe&8@ol<0 zsFwkA`cP7H`HD+I<9< zwBt$N)08B$ZPaO$l-{9t@CL@qGo*1ir1@zi>5}pga~ZhF++ToKDFBjWZR$N7;u-XXaBtUbqoaqxuCwbw;Bemrx>l;&21 z6hJzWtUW6bpB)(&3w#@Mz#pr%R|see2qMG2c?{t(X7Dk1SbP)lt^GsvfP!9Bi;SE2 z3+pFd7Bv^3Ls_tN39SxSjm^(+tk>dC|nF*K<$nN8Lcw*^Yps^^Q+1BG%c{ z16KX!J<>Yrz3^bz`ZYXIvzEO-Q_Nx-sbS+5snittze*B`l&&!_>M*=vBJ;^U<3Q;o^_3r#Q;Z&B1G1`Qmte_ghygUWVNEZ?uu0 zFq^AaQ}(ZbO|LPFq|pX|I=+6MsV_)ix}D)QdzkUnWQ|o*r42?+3MiN zd|x)swT%IA)BuI($j)=THi~ilz@Ww>=zME7HwnWq1fO&Bj&u;kfg0I)sWaT0*HggS z{oG-kK9jO3iizhQucJLgkganQgMh3(7x2^f;oHq+M7iJvA z?PVam9X|#S6rHa(cdht7(JMyWJJI!3UAbc6(R#GN7mEu=p{V(+Q(Pv>cD-R}5hwTp z_ez8nHcABFpCf{$ zql}QWV?^KQzjS4F2w1cG(JZ;!g^s za7#lcG2oW*VB88}wezLTGy{2*J`AYk@G%_6A*{NO#T0qOZBb5VNuSFOryrpQimhebnSN0sD1u&WTuLlIg=oiyJbxWjWtB|7@Df$USb+zp_7ivL ztl&r2K(T!729_4v5nrV{A;bXjW&V zmiS|Pv{Z1zbVPW@#z+vVoS>l^?4Z!8BlnqcI-dN-@!NUGBk3du&toBE@}>Wwj3dO3 zNWJ~+;V{883FU0C$(+uY=ZU8-zgAR{P5Y_P&=sX%XEh#qWlD|>-FLr#1Y3IIwvw}% z43m*7R&=wSnjl8pP%%=Z)`COtTf>EG>m z0{FWT|7)ibFPX)2nz?54GIJ-)MDKOUc}@6n3iEq55B(QL929VFF(FGaK?!&{X4MEU z%=|w132^6Y6M(b%xNv|xyzzuG^qWE>_AaigT$*n%&u&pMOJ?#OqdHBJ%akBvPSA5a zu$!AxPCne*9`Ndz1_`FrBk^=HEbfSR{XTqvz*I->#au?oa%b-B99 z&nu?wkvY>OidQlzGoI(U@e6>qcCBRJy&!_d2+7)T9+~lK@a_09ctFH#$!6?=qv}vZ zth->^=?RgA%%Vi;xRCPRfLgD3!tcn3;5Po^wZOGsU0e-CkS~m*3`ubAflMUs?Mynm z)bR$1WJpj-4E2Yw1LBWIbqFDX!Q*NrmrzU;_X z7#t*fy=OO@A!WNx$An&!0)!_i<=HkO?<0Ff@y1HBLntX;M}SI3QgE0UBKF7diSE4JxincqWzo`V z$#QB8Gg3)3+d3z70`bpJ>mIfNtf%m)64~t{vYAUE(i&yR;~%%9-)Mt$-v>otjvt~K z#MK=QEW2yScta=1(2I&R5>kLxQ+mpTVWh`URh?4+tMQM`O6xG4=zF8lbv)em;!JSM z*}T6Q9@iHRWD`M2qv%686?{ZeC|F2P)g029No&Hs)A4bAh+ldC^s}5~ss;Vmm#W#Q ze9vhd!?it&k2+AZ|I=143kf(Mf#q=hc{GgtyIH!Fntv~Kk&Aj0=vWlau}A+f2okSO zF8@hIw1hAv4frk(`kN%GfSImM;jX^oMBzkGGMiesy2kAz@!@Z~g4Sry4u(uIh8YqwU>Zo);NBEypl~P!J{iZV~$_7%4h5>fhyAcsFHakybBt zmY-1f7t%53Pryxn&Sv~+_-NsAjIrmAwXwJ3hwFjjphyx-MMyJW zbh*COT2w8LcWbIeNaqzB!V92NF}xV7t?b;Y6j(Hk;!ZE>4TQV&ZLcgQi|;-xOQ?j5 z;bj0`X;k@6r<9tiUP4K-G>O%xpSkT=UhEyLmv=iiI`vNF6kWC^D+y&)6|xV*bQKVb z*tk0SP^FylE{1LV;Cm}633aV zW4Y`n?|hfNsX9-_F4&FUK zRg2r{<4C>6R6hZjaNifB<$dJ&Do>`?oEm7GhE*&&)UDQ%^>+LyJs_%q7YL{@?QpQLLvyDs3!L?Fg%^d`Kkd>-MtIId-55AH2dsHiHE^un&Whio5PJJ zw{y6Nhj)}!&icreQ?6BiC`Cl@WE}s>K3g`Bgvf(m7vCuqg|bPOiZ@qH=)R1El<#_c zTo*80=JRJat-LUIzaozH4COcrN-eI)b%$BHJnKw=Vj|*=#b&%X6U39C;m^E@X1JY1 z(mYGTmXP6)w1ABcOoAf+5aG{@Bq8?=qzNEB z5#w9@vLaahiCSz=*+X~|ky24j-LG8Kr@9!zES+>170$IG(3F_slmlu$a)+JVaXg>| z=#b12Q3vZ$IM)fb0Dnm}N2K?)pq+M!Wf^XrjxS2Wzy#9(ISc3h?T2FYxQ>Go$NUjR5%+oNxN%N^%xgm0a1ixu0v;h_7S^s! zCqBxz`35h?we>te7dEq=SV@*&J!EeNZNx?;4f>bDL=_%3l*OF{@-q_y=IG*4Ez zs#w4-54OFyaBCdLo$abyMheDQ6{{;RsZn=iOW~!>Qsq)0YLE03?cw=8tCFZZY13ef zFin?C;S50nJ)3yOBrw9MSg(BCI}mJI!XLp{g_smZZw2jz3PL+F))y1u9wXY$>YCLM zz;dW>U}0nVThaQmFGf>rrJZBNrqPGpt|fRoZSMjq=fMg+npIGP>B*@sW2o9OH;gW$ z_#P8i0tV~HChca$EXz@>czB0G^CSbTQ-P{bF zA0l5VF*$a>A``Dvz%`tWkBXWd4B%T1>?Nf>?jM{g!LutFp`>cc2XQlJGFNVsMo-coGct`b(sA+Vo`52*chaO9pT*B&Oq4Qlka zQZ&Q|u!_$cN%2b|9d;Q05~e_3?^cHN2y%0D1tp3(b<+^>)R-?*wkrCl!<2Miaf`h9 z%5qCj1JiF4xvsl!HnU0GE-lJ9nO#p?EZ(cI4|9r95|;_nD{!(PoJbpo-Lbu57puhq z0MFprBfM#Wm1^3TmZbXFA^i5onfC4Jz8{JV5i6W%jY_$k(!p(ee#C|`M>3paa+F66 z^AYwY(RA>c6AucCGEooCRx+Kaj$9jlkVYqjlVjT~jWdA-6sd0?)-B$$N zNIh^kC$26Xcsl4aJXy7ukc}*$jqzDZ@*AB`rS+!QLk`}f-t(RbopvZ0q5xAPC$10q)Wy-6?nglnOo^j>PXxLDX-oa}26#Cl|R8tC~%QY)JNjL*3-4uHt9_F}D&k?fR?A%{6 z)iUD=-P5qOpRoz-bew$gG^=MMh9mOEgw-*e0b4#!{Zw339_ooH?_Z&P5(OZ8f z5^l$0WCYto-?xfPCJqKJaAP0()l87W_TStlqoqcrI?LnfRqi#%i0lu z)UW(z0())Y5W-~D>!_705H^DCZsI^1lZqP_Q>L*M^D!R+TD9PIwyButOgK*4S)ugH zJdw6M%}s&!?0T%$2UV_MMWwNr;Erk06UFRRi!sXBiAHV|>e*W(oNpWk{GCSM82Gl$ zkL(o~=H{6W8Dg%_i)(R;*}Gx(&tE;SnlCA!=g2u%Q~#n}pd3GLjZ{px51xmMgivdf zaZ{&-Q&M1#7nw8~WuQxSNr~su)b=4c{VlpqOT7HsL`3CIA&$rBWug#vdt;eIM8mPA z>;%%}8&m-)*LlIQ)?;9xbK>G~#F$_+Yid0^ssW$V6pkGs+n~sf5n(%@!K^hc;$(0T zb9OepGH^Ka%cJyC^t^%BERlh6csQBwxyMv6=fs;%bY;rZ8u1}anMa32mSY4z(}Kxd z^h87>s9vt2bn^3f7U4k(T_$Hi6gRdTS_ALgv?&38JASwxsQNO_NenDlmfNig-I(&y zgxhQlEl2x^T9@t58Uwaya~}&6yn2MaZw091LmGisEY=(X98#U)u*IU%v|yCz1#hsc zEX%Go(^r`a-S*IBjznwGwFq_Z&}9mC{%qiTh2eSLDmDeC-o^WgrYxP5P8MA~ahpF` z#;Ecb7*_eJHzN(L8laXc$?aca@uAd)2*=Zul~7rQ^W@iU13_aI7;EoOn?mr%U}nCQ zfvnxMPm`MIM~KuX;P@P%0EH2yuK)Uh1)_i4o58- zMI!HQWboVfh;3BQ&`?&pXfohKC^4oHbtFLNg8$xBCjFtSBn_(o3eF_n&wC?sNCpF8 z1MC7rZJDl@phJ)(FnX$=H%g;HRL%%&a>j-i6q#KTERQH-r%(lp!Z_NAuiajJS2)I8 z*iCd#rSOzu$8BXy!LuzP8+>kz7ZZ*lM?`W3#LAL^%sAco5M9E0OiF#nc*6si|0$^* z<9Nj3Fi@gM#+;Ms-2PH}gI=+{i;1yqX;b#MxHRtNr!Eb1anGw(M@{ZN{t)$5Xi|t{ z(a?1uf7-j_Jl|zZ%^n~2pmW0zzx`>B`u1es6GgLFK-{)=vydFdiijq5c5}uOhLfg8 z_MhW!{=n*Y+VHZO@AREF*%lGexk5i^1KsdEhWXxtdf4MNJ)Gic4yR)yQ0O5bE=x=x znH>U(_Vl1kz2J|_510{Hu7l(;3LO%r5;xibVaD=u(YU`!s8MX%G>lt$_CeC4PwIq& z4J976A#JA-*@mva@slVu$Z&ohPdiiqe>cEaUGz7M`ByP>;BmtE z9(~N3ebY4kUQP}9Aru?PUj<-7V_>Q%Le*GtBH1(Dyr3EvI&DAoAT%;u&o7*zVj9QO zZYsW=e!w2M>0f9SNfqdYk62qN+OAP=$+aLVm4-H_TskerD`~3-A{Lv4wBQt03)HPB zaV+43{r)>!5mBORebewAU(0?)NHr1v0#`rj~;~K*e-fbvwyA+nhzyFz^xhn;fQK&d*4Ah4x_&m;>JAES!&HF~< z7MG(iR)6IM8pAyk1Z|AKc^c565b@jaPEe3=x=|W?;O8GSy^@=y)823g4o=WT+KzJ} z0|ig`O`7HD?f7weU_p2Zr8F(9o`75g38-S|3CLyBVrnV77(^0Ixw}yTw3;vki651HBI4v0#y(7csq)e<~kNr7~xx83Me;}ucJCS#8 zbW+qYOd~lKCj#h6*i-sAO`X&3G*J$Y;t(n+Dez${5K?{)=ZapedUOG>R39A-#*-5F z({wD2JX$Sxjlo^jg-79S>x@u-TSVrK5r&UZs?nphMdFSV()sGpg~Jh}A2p_$PHQ+P zVT`G#8g&XP474^~o+rO?baXyXTs=bo!Sm|X)R^Ab6JQ<>2g-LjIQio_o_S&L`NoX3 zcUU@_ppGltEPwoQQ#oIlrE}XHi}-wRqVOCC&8^Vbfu8?0x0G!Cij}uR;4{H(TU;?> zK%Y$!ocjBpzNB05;EdFuId_6KgsG9Ha9X6?i^LgLhTam2&`XYWsiE7E?YAC>{BVS@ z$erB@pxGi4Px{0;UJds|N`x5BQ8ih!F|tYX)B$5?qvTkmV?qMV33@3%B@t?D=*T1N zz>f{&)zM0q^AeN528L>{_VAcjo{_Tv7gnLiQ)`E(3yYD56BAZ8kXAZ8v7VET5waS^ z(m&V;F`ajSljQ->Q{!*P58MM65zV$e_yx@cPf-z6nu(l?mZ$ZC+)x3iyvm5(-H%F7 zZbYbq)sYst#tsM}@r~|0ytNwNDp}EZ?IGfaTT1Ruq$D2i6}SZ|qA9l3;)9e+!(p77 zX%y--kUHAO>QVLhP^OX0p9j_%EEyC0g2ZD?M*Y~Opk@9rrBT*L%yQ~Hj>ni$S%9h2zGVpY?kNT}?USYsfn**U5MDr24nGlv z16KTeoQzN}h}Lsn^X>G5_rT5h!bUjk|1o;uvG5Wx*5@9=#4E|$U<~d?#^TDrf@(3e zJ0>ACv@ou*OJv9i^&Y1u?8DKy;DieSgs|Y+xPbHsV~Hbg3_QctfJ?5ESzeUg)qr(R zc?bb-hm=KRIdH;y?8#YG_CT7w!W+ z>8p;>(eO1E!b}}x=tMQ#lLEMQ{xJpIkR@rc)^Z2>590 z;9|R8EOKNLj&}SG#T^UNbs?j@h&&E^-SEEZ(tP2(5Nl@lD-QJ7oQsZSy0xu{SlVt7 zN=TDCIbQxh(E~SPjJ<0Syz%Qc*3jLs`#PGhTBMO->~IKC3J`)@z~?l6bA_>QCaJrv zY)STg#FlxZis2pg@R@zdFc?)ns$tAwd_$LD<7muAYe8-2lb$0eB{!c(U9Se)Q;;dK z_p~=ja7dk(Z$q(z83u8?;S3j5|GQ&7noShz z@h_vi^?z6%xCs5MFV=YfZXX=KQn3gw6k1AcsU1(Ii%lVQCuwI)c6oRo^H>Ta_5A5B zu!XEkr03Eep%v6?9B~n|d(SsPJyf=Zc%>;4Z>}n!eMjGOBYzr<6hf>hilHQ0(V$8f z3PH$~)(hhR06+jqL_t(y!dThLXvd(`O4f?Np+IQ#3%T>Unw$O0n!^eHeD>|7cmBx?`YobC8ZH#8B@~q zuP6~e$t%BC0^v;sAv9c6qrNju{oC<$8mVr$www3aCpNzwKPC^T58ZJ2tZ&F$+~1-n z;EDv%x5c`)0KBv-o@CQ2^&0t52QGSwq6N}!I6igHa*4~c#i%3ZD@FkeKSMh~OVmnc z2=!oW?50Ii)+~Y{OvzBRJsqTW0+cV=;;(UP1M58HxOV1G8PHb_55;Lp)rgh}cii(v zzqUem7*hkYnX-ytY%DiMD~8KNcnsD`-=3uBjF_IvS>>RCjoz59do)$Z=HeN!F_QZo zV+HJOK@a;#ZWQgT;@eGSHMWLFkh%JLO0iZWvuwKN)A-~KTKA8N`WB>>UY!X&ua2+f z;qJ?X+@m9Oh!m&^NRH>T!uBliz29#;m&`ZV``YpU;y;IsGdK7aq|Pk!sC zeCH5}smMotU%@s68Q))i4&(e?BVTQmLW0g*SXj7q-W-c(f=7j120Vex*&WWuU?RLY zvF=7SZ?j&j>6jv>y-}|i3Xbs{v&LP{H@2BP#?#aL*c*7oJr#*TPGbkwDv>5ZI3AmU z9;_~m(#Ji>pcC2vQoX!2aP1g$?tcfLG=d`pLdZ=eaDL(2=d$MmRnQDX6@vXf4kwc< zMs)y;Vmito=>dYP`P0xfKOI;Cfz-a7%%~WA9h&JG<#uQ<#g-1<;2Xf)$HVOIQEIuy zm~w-U?YfWn=@LRD=sTtbD;`s*&jPy5IL{pL)fatfQk}QshwOoy;h(<$?6c3x?4qOy zyO1wJ7g`J7qD+K%6i2>3Sj?`pNj^eIrZW^D3*fhDTaQYJYL! z`i^79ySk!_Ta|k+Z9Jpt7#akn#r4v=ZN-2|Zv~-kKG7v7b>_)PSEn9bHn+o|xa*G{ z+RD~uWwxw%PW4*sDN-hzDx)E2#Yk7R+1#BS_hz}3AO@W0O^N~*I<+0mkNb16#v+q5 zZC9kxWbLXd_L$er!|&ezRv!dYDFGc%M#ulw@ri=o9=0eBqvr+eF&PoM=w;f;n-b8* z8_poaX@R>_LgaF%=hA$Ia&p}*s$iZbPhPtFEEa{yx)qW+%QHE+}daN3O?Y0io8XG;8INH=JXS0mNn*kP)M z)Qf5j#-8TnEE4KgTEoSO@g2xKDZudSz9tSQtX8icB!c!ghUdx0=ePSZxfcN_FxYWV z`FbinULr7MfEpJ|-}(56${c@v`svSp^6vlo^3z}b@?AF>b(0eTi66NK#|I);I{Fv;M2EY31^N#nw`pHjE6r#7|2kC)@ z=O@4aTR;E&fBVGtd3)W$sgx|`j8=6^<08n#w%96$q|;R_y-UV(PhGeY=iqD9DMlM; z7g5+LS^9|k$8C*$Gw-sSln+95E<^*CH)W`JA9`plKCOk`s0u3fh()%me9Vm zcr=?9_Ey^~Px&V8jP(!$wO2D473GGGkZPko=bWsDu$UsT+pAka*cJKdADIawG$5B$ zM?1&tlr>Nt?vieed2^GDZ3nDAs`Q7y{ilBWv!4#~cKXqI;CFxe)A#@TKPw)Kx8AoS zdH6so)bKqYI~HdG#@NDj;95bXs@v@9gQO~P;?_6f75B1g9LEpQGSIVV9lsKDi|{Cu zG!pq$eIRZUWgm1UXhRr24TRXq2quI?omvswL(9op<2(x=3b&Fa*LDHKl8!@$FS9NC zE;gp>hjy&a3E?gujSp#fwbB+!S{!8JXKf>j6zxYc4FRtIaK_GzBvQ67z3G4YF%HM^ zubsK8w*WrSay2IWZD3aS8YeST6SzX5aUP1eMh{9k<7^T^Yu2dFuZZ=fxa!y;!K^jY zxs4aG7#mB#sLfJiu9J;y9&6M+pT&HH=p2t8$yM~jxgBVYT-x~Y8S1w_eMA>9-2cfZ zKl|BdfBb*^;-{bgJCvT=HWQ^l~7?%iI4Q#=l{Q7w{^7H@64r}`b?!oeGpS$Sq>gDkqpG1QgE)7Gs zRjR=!TwqY6%c-|#all=vM7dPQ;SPl|*2qpGVPo6v_&oO~ ze&#Ou3;XT)hv|Wv;6MAbpa12b|JncfpZ~AF_(gA?^1F?zz{Q0?61pABB@tqYT;42T z3al-8{ByS|#C4?a`o)xf5%QKp2_D>^VbT5GE1gz$j0YlZd0l^>4vnSkyaMtnQ}ogs8yCa)N4GX4)u8Sx6y%5 z@zphmxuYjOS>w@RpxJFfj1!I|@#R?^+3rm4it+JnIZ920t5bVUqd@j4JhRSRbY@uj zV@hr6CLZ1XPE>6_BEO-OI1YD;Gso zocj?sgA1~{MO`#cN}$32oPEDpWGEqHNOF|wpL&21IkIC5c!L+fp^s$)U~5O1mZv8R zV~W-h(7zm>a_%IJ`RtYN%R%=GdRb;&%%E(Da!4pga!R;c^t`g66@uQjT9a3kX18^u z$W)TZklW^*>UgR{*|n-xliH_s=aF5ZnhD#=8-A?YIQ(R#s|%Nvq2OVhtOi*!FCeW5 ztXQcp3}mUz+Bc>q^3BuO!@o(7z*cuq1W$XC#=cW`MkVSKTYN zXEL#EPi#+YcWm3XZQDLQ&-0!4_r2#YIBT7gb+2Bzt8;fYs%lqVb@krp?#)mR>fri; zqpt%>+!h};4-;VY6_l@^dBc4L67h91>+3qcV1bHjBRdUM;iF)q7awS7>ObvV=yd%k ztx8(+FJw|l&GI8#Bi=g?H&Tw3k2?9q)5acIylywz4>LV34m%zXGMjIRbG~b5Yzt+| z#i4#|lg#Bp7PTbMqw0^CGr;>`g&*8ce-qL9X_z;Ufn_-`sYMAMyS1s^zl;{KXPSz^9=X2~Vnl zCzE`s7F#V3CzR^jQ3&D3SES@&C8R4sd@E7MhPJp*55V>veTm{wo;n6zaEfw(GHnlO!JsIM=tRvUUkx($OO5=Jw0CJ{^F6R`8vl;MWb}K?zy@3;>`sQqICp zqX_dYb{Zx&9-s;#c`GeqJ|9;JEvX3+4(eMGTFOgegnZ-2@axKE&0B3}Gx<=0SSZ%~ zRJ-1(sYk^J??jnakRXJ|TJ+2=7c2nb0uH3VH70P^R%1ks!C{7a+c2+X+< zfoQ_l36hCfZxF@h56NN5fT|%;vTlD-MgEL6V@ZaceEb#qW`cf<>0B(UnEu!`2EqX1 z74K>}N{LO&Txgg+U7F%KH~4_*FcjucRBy zFlJ74Y;mk1kU*@qAo1@q6mjM^si3C>fw01paBb*R#*VvY#mW-P&5$6V+T=~O_LSic zlu{!&P6%#3;oO3@LfeJ$ba2fTdyIqk{z?I-P@IAU0eaQ)XnmS#=lR}P3D%D` zOn$e7&PB)b0*7Jfp0dR*`ott^={NQqN%0(2yL~y8)k%x$B%6Z2vc1@FJL&R{yrX2q zP_J@8aiwU$5V$a^@JNBu7AvxX;rVk9ihm5Q9OfqJiesywG9jG=tct8OR9yav8DS$T zwP^ZMp1oR-1T%Y>?EQJEy+883GXKF1^Y4(y2lRb&rGlh2VkowB(41gCf&5Iw7xg}3 zHkf{o%Kb&y{1_z$WmJ^wxK{2G<#-WFglR(mpH&MFlDtov3KvC&fUv@ma)8*4!jDYx zuDmnhgDFBe+_#+&q~~j8idkYL3dn@Beq%Y-n4T`a`?v4Tzj%NZ+x#Wsg$KCT4 zfJBkKKupLmdklGaxr(cF8I5)%hcJC32Q%(P^0FT^VKXn`-f9mkB|3BBuBS8n(mAs)lM6nu9lNp%8vdkI((WCR~MmI{da zxI2vUu)Ok^n&O$_;P62{WwXSxUqt&!;5p&;5-B_!BLvo-%y)gK{wlDs*c4L4dK9rA zF^3!h1#Hq~qD_?a2>I}2MzMy9&J?lVca9oEsDv?A7Gu*lmLoh zkXD&}^3;-h%7K*e+~e%fBeU{%;vFL(*pD(1N!faxZj$$>D6GP;_+2g_4>v`O-xJKw z7?z;%FAJAT;a~&=5h!fkp2J-`9KLf&kSW5_@V2{0y(dWC?^oms;Au!9PvZ-`aK^a* zaj+^2o4TRIgATtQ8K07P5FVt9fmZ^oUySi2od1@>%DYQY3R#up75Cp~s& zOiRf#tYY1fUQpL_8zLPTzeDg10O_VKojBkiVR619tnZVLJ6#&lRW!&sEV1WV&ZJ&_Xu(0kt6 zdRcaY5n(s)AuB?S@RFCUEBNA$1~0iEDqtZ)w_*X;dNwQUZdewNVrKs7v^6)J-=TrK z{kE(+1_jqXoZL*^puz|{YV^rMyUE-ShbtN#ZX*#8&17;b2hgUf)7>D`RHsHK?+ZX^ z(Ckp4KqPt>^bU&EzX^X+Jl5`Gw)`#2)2>X*Dg9f;`@};Bvk*5bPTu4c1&!uct2WFA zRI$46-IG(l@cK2(-jf@OU%CF0uN=AyJLDw)V}@VVwNZ3PD`B9q)}l7 zwNnsLz;SzAP_gz>oHDLNQ)faRZRmYro)O*Q#-07~Zt8$nZjZ1(_qb!5u&G90X;uo4 z0`wiJ{7D~oG!J_c6DncO68E@42FU~u6fc7uYwmQ~1Qpe@2R`}e-h&dO{i^vWLAK#PBusCWFNoMh;U*&g%h(`}2lCp87 z0b#P9l$+8~-#{cvh*68azs!V!uyUXWDqfxl->3{wsic<;sulDyNjV;6eiC zB%>vZ2}1I}6hGJp$)prIO0HaGZ?f;r8&og2rCy>!tJrcVVqfQA_y$27+oB$Zf?$`3 zEC(b5Xn>JWE@A?h6D3ywpna!wQb9j60GDI~aT79^5lufMX~C7<#41^_kV+Gw-#fay z16-T`MRS%lM9?U@Be%WKOoUg)zOp#IJ~ns7VC!25v2vhL(Ip@O-)s64l7BLYsqrm}OKDey( zEC${{&YP=ag#z}9ac@|=;@KDGCxoaxOgr=0EX_uf%h4eh5Q)oc1x}G*HpyeIgE^Fw zXZCdMf|;U~FIo7%s-B78fd`Jv{L)1Bz}ZK+O8=Mvm$u(R^M$yZ-)oj`+dSY!fL;Vv zR%Qu0ls~F*Nls>N5f=x^!tP%IsQT-_RFm2&DYmjI<#Kp^88}trPVg`HFL}6LGKHV? zJ(N8JO#mtx>Y<$^!HKT5yw+tt&Mz1q8Q}|Cl~4*F)i27I@9|H;uO?x(S*$d926mp7 z_E~!hSe_&Ake=$E=3WGYG8cHo_eV%sHz;`g&k9(w)Spt-k;8t^1&s9R5aPc7>%!-U zt{UXvr%df921UmI&*fLh?~5Rq#Ut)Z!ap+nD-HxEhCiNQ%eggD)F7T>qLLG3C3_WH$Pv zvfT%ts7S+eEEum77Zvq6ErpHL1$(Wk1_m6fQy3UugDt&*XC3;?PMoMbxm%Vt2V>8SzMXTHPT-faP#z@|~we^{6YZbNv+RYBf&#~B-kpHsAAY3w{a zyVadMX7yjzU2j#?4E6OQx+};DH70HO7rb*y29a0<_EIO*z;vRT zYJR?@>}p!G^ZLRk_qBGdakW1b;uMIFht9`KLh*0f3@(yJ$ubA_fpiu|M#fzIw-!Ac z3Z=gt;?E=Fhbo*_BY*?;$2JwIt1kk$d2WJJTDh^5a_y17Tm6qk>*Xdpi_H$6=}g{= z=p0eAVSENl;K811ui!0OdAVhtyD#I$;YMf6L`b=Z^IICXF6raU0^_J&`Ggy|es?6(2chJSG%U{K}H@Gm{Rv=wrom`vb~4VIYwD z4272au!T?O%PO|)p&vNM=gaGCsjBr_E!XWXBH!g&Gm+>8M+OKXni+tT=tztfy02tn zG#IiOA6vajEP+mYh*DJ?+Gl+K2E#L1-pp5CTfuBGjD zj#}Aq<*ea$UPm$edVhK^^_PBkWZhlW$_U)Me-NL7S(gDB28Zsc=)Nob zXjUwpu+GTI0nH<0Zz;kP-tLne`gOF_60w!gID8Kk3-|YfyL=3496i4Y1j$ikH?b43 zKvRQ&#lx~gf^px=%+6CWA>JZ)`4ziI1bTep;M$j9P?Vp7f@pv<0AJEPT!_wkKswO(RD7q;x>WpubX81|-mE_t}g=ysxUe|< zoVilH!@YY_3IMQ};CUQ#7|htW=p$-PO6=9W+24xpRK1pscIP4p1p zqK{^a1_RL}5n*7a3&<%cGw=Er&CFg2oTo*0xA}W~3#5`hxozx9RcQt@*cOn#-;J{= z%|4#PRYPXk9blSduuw8HLln$L4Mt_O7JjlrfedDHxDQPUT!GthTO6#ZlrA`(EPeEZ zPc|-c@IGzT{{Ai0kR79765-o3g(Pu7MLIUXXmNW;quF3tkT>ydmiplhZsK^c@$hE9 z)WvdBNms*doF%E=W^rI}Ehz@-uS-Re2QC>9%1*Uaz&jsk&^oT^!nK$hqw&;8v~at- zVtt(!N6U3m4yGb{78Z-~k$QAJR?xg&Pq*H1@y(>`kx`rR)DI;QH<}=HOqna?wOP_n z*Fin&j;q|;BpRLTxA;nRD^(`dU8iRB39SbE;nc7(uBRXpGH|Lo>6)L4K2*+L@ zZYIG?mPSUXdY*4GPEInWw(^&{>sG+?kAd#9^XcrArIa<@ZXu-g@{fQim0E?@_46WY z+3Ja77q*g)PC3ic+r{eT=I543_|w634)dA( zOJ3JYCXWmzE-sTd$Xx&cARmA)b%csSmd<5O{lZ2nOu%X_~+@C z;>%Y9rI!yEvyC+^#v?BAvk$m20&ZW}*4lv4!BO?#=bwaxkh7`Q8nfMnMqQtx!iE7u z24Jt?hO77P-k#w)UUpuA3RDvq*zwWH#>8kpA3^e%aA~sUI?96hey&?qPh(`?O3-HC zzmlZqYS8;K&cY&lY|oT-_k5$% zRKpW3^>0LV;s=u2X*xxxTUSv1??UIGNy{+FX2G8(g4RwCLRIMN(De1LFAe@hi z>b3b$_xWNKQ$?ks;d~|2=R+e;6fsiHw8Lw4W0LQclZc3_!c9BNn@6kBIUXppndm<}E3K~&MrV^l z)nBye+II-Qekm<0d%0LGp;9Zem!ovTfL9Bt~)&AZelApdHJgT-$&DFlo9yW)Zj@KR2m>bQet;pFUj4zTj_kq}0?9cur2v-_0*K<|g?*Y9_3!RxMr- z&oa`}Evu8Q1iDF^ME+DE@;=?X+7$H*vq_n!e8)kF91+xu2iR^5jUg$hvQH$j3$ zGwj8nE$t85N4|4qIn((v`!RN=rt4Z-vEcN;VFHi2fS1?5W#IXFewSrD>918B*=KJ{ zo;s^vmrT>rc0^dCUMA{*w1DAyz(=S4yBuBD-nz46y(fth6_ z!@-!YA#?IT2HQ(eTIKOw6x-?M zMQs<3gC)m{Ib6y7`p9l&4EH^>N)TN!M``8D4j$Dq@#SU%Id5+Ppxee*($#ISJ`eJ5 z4h)qT|AKH9o^?CjZU2ml9vmE$<@Au8(+edM5RAjAO#dF7GRb{!ud2GS(?1#%je(7h z_jr_t_;YEs7xBYF@8qN7y)`chANw*DW)MzI@XX1%2z+-U-Skpxz7=a)akIwg)#K7T zE;BQ@ncS!Sa)H4l%5p{@LgPsG0+z3qYE07W zlEI^{?KR{T?jIT*7(_~{#bz&ylau+=!Aus|2MauYq1y`u+kJBknY0C)kC9(vbmY`S zbb^ngg$X!OoF1pFp=$5PVOLl3DeRT1EVe&rq|m7QDWMm&Jy1EjQ~Ub|1_rRXU7@cC zh~9NaaAsZ#3?M%vzC7`DtZnaYH5CHvySjKEd0Y>D{obou?#A$pJMRy~L;^`1d^EYe zd!DC4iLX2iUDH;5c^P>0CEQ4NHumr+F?igMdZX7@%RLUkHJ0N$Kd)z_JIl!{lNsL0 z9PhJrw}Qq~cx+}1Gt_JK*Pcz*><(pIi{LF+YdtQ1RC-p~Zri8q8CIJfj!XGx{j;nA zzsuPLVcK821iZVK0*S|)O7FJ}+h}h!AcXRk7yBG;x;F9oJRiHevutZFaLLLv{hdn@ zv2Z&#Tw6+M;o*4cx|+fE6^|F$vlpdd`<1`LizfUsDvfo z`h+663?7>YU>@#QTGF3tMqe+LNPL^M#1>mz!VAjxn7~JL`${k8pO-N1v+5Gye3UiF8he#fcEav$u+TtLE_FUEu zz8J=MFyw41*#H$dJ=GwBq}}!S7ODOn4mOQ=Zn*Jcg)$NbkFk2AHAW}dXompqtDpbN z{n0W95tG_tXjVP9G(Idk#G%w?F!LMb(17g=NYiH?>)8H5c5dGIh;#G52!d zP^1{G$=uvymY_wawV^i{URZ51bKS%FLh^LI^t8c>tJEA2^!10Yr_oE8{ZIQ_1n;|T zlUdS9j+jIhCAD4Kg`Zu@F_5?S{#C9u6JT-X{#*n)XqjhQV#fj+fd1%V`juu%y2_;v z2Rf|l;bEF>@u!G6FRb^M$I!?~YU*Nc*W#+|Y$SxQBLqIi1_`Nc6%K+N^dN9JI8(>Y z#S<|^Z?JM4{mb&1fvfvZm*H}{^cT|SA_6(^QJ4#$Q#ce)F}f)Eak1L#?#?2^jtOGl zc>54%Ws`40lp%VEF`~d=57!e~!Jn><`qUvx5tk_lo~K5(=YQT7#`8y4eS_l}ToL51 zz(MwnQ%Og8^4o#^)f+(KQQm6^({o2gO}RB)eiLSehJx}V6qjwDZck|v{4b~ktqm58 zSMN9KOY#paxBHE~W!QrolbM;}%wz$v_KcQkBe}-JWuW(wFK}~ino{J)y6^Ean*Xr* z@W?Q5#j?oi;K{cN`%)T_&$u<{p||?z+A{rT(9qL=n(*8ZBkx=xYi5eIc!4h0(SVJ) zZ=u?u^}x<<847_mxwBa6@L|)NYy8*OlbL2FPG`qYAz9gr@x}FO`+iX^+bLfGuWEOD z^``Qg+?QJkC8kH5s-aP1Pk5J2Bo@7pK~W>$m6A-TJJdc=k#0SC@8NGfNHwcQ9T81# zrhw4^6z1M(mK_L8VK-&~;+>8u)|XoI>DNKG=)}Qn-C8hTSv6PTCTINWMI_I2?+n2< zjK`34c)wKR9?XH9I^Vf@vw>zBX+~tQ-xIT0)gnxn2eQW~UNjz2=IeWn|V{flAl_ zU^T>yz{&a54;)g3rQ(?H&2i(|Brgq&?lB}Xl9sA+B`q~-+c5N~p8m(QxCq)+U_NFz z1p}Z20Z%Xz$04PB#{I!!E_Xhhi8-0aCNP*A>)QVd;?Z)cl(F%{$K9RKGbBl1KTN0{ zk?7Z+2y}6wPc0kICRQU0#Zokv4Y%HVHkpXioh@vA! zejmR{p&0UM5^Ly)0W)P~;Y;3i@61lu(?1y=&P6Bp;EsG>AsTJg6;u^fOe(DwDKLeq z>Wm?wAH?mA(0TXeAww>U%I9-FJk+Ul2x=J9PPDEEo?E=yKyyKLu);E1D;gHiD-gFb z#0C78BSQI+zf6L!!NE<-DC4*D1CobicIv7arGR{-DRd|a6>MTuU@4L?{fV1{U4%Rd z?sEN36^$5@ZXh!hTNMojuIiVTp|YW!-SqHMpr4xOpMPNEC^--rMIOZMr)kRXjOv({$20qZ1^XB|P7}>Ue4j7tlbvp# zw>G>_Z=;C3uQ%;-9iEpN(Yko-`O%}tON}mkg}?H6QrBAcpLZt(hu)s0e3Dt=91OC* zTB%b*KkOpbGJwHbN7Mz)Uq0QK_5JF;F&1@1_}R@c4-idLgCL%0&H09SnB(y{A%J!| zBXX+icJAd!zm=GZ!`Xo1BVOO%Xt^9FU0QSJVWB zvU-CwVH+$%+-DJIoUhD5Ff2L~(hCOLHu){r<>33k^-CXQpaJKCno|?@kRC+#w=h4I z^u0Co0D&heb-b^(%lTS3$=sGyNZV?a<#o*(IMnnvqofiFz^h1!p!>cGj(*C6b+&LY ztUe-gz;G$+J?XZXbtckXxtqDd8}i&bFsH%~aT>0|){wEniBb@7t=#W{3~#jAgiDfg zq_y{6U_#!6`OG`Rkj;zfxSWSJ#l?%x=zA!XhI2A3_ou{WBy~TuvAAipswq$6d~rKsB;UsLPF#w=5{A z*najM>Faqutatkd>iIA%n>XAYO&53tJ71Cigi$1Elwhq8r=TR?=&(`?-$Tn`)lxNn z;Ii>n*HAEEgwpTe)jVJE{%UBx%t*k)V{y8$K3h0(Hdmq}EWDK64NhV)WXf=vTujjW zCDU%<3a@Y?gUdwUN@iRIg)S;0gqn&fjn8Lzr{BW-)EtEkz zltlCK`3U{kT`k>dQ6`J{Miu?s6YYV&Oo0e$({=}AdI z>Hl|GA_pUTaYrCWqNe^^CU~O+Go8q7&Z_ZN(lig6y^uCVVY8o`%3Eo=!Y)IT0;1_! zjb`;Frh{wCa5gQ+E3z4@h*)c%6P+7xJ-0SELn9uK<#L^$-o`R2DsFD5U%1ZK(}LI( zoLNF{Q|QFFxYQ7e0tRAys~B*Hx7cvZ4hT*dk7zkcHHsc zEH69`n=;tB(6gHlqYnZni^DRU>O-j4<{dc>O8$zWwTjxrmu7zj$`3+ zF4W&{dh>TX=W=08vob3=*2IWw5md=SL1Vc+Mq(5E*edGcQrsQHr$)JEI2>e>5R;-} zL%A;qKUBd#$oZZtP?zf*Gd-lxYXdwfI<3+txqX{g1t#goEgU)ObEK`mf?*?F@vIG6>+oEBWl3CjL5IE7=`V zc;DUaMZ^~0tTx~hv}9eKN1^lBuQ!?Cwoo?|HkHul{j(bx3Ez1!K&9(*>*E>^gH>lb z*Aw2fOV_CF{`^#Yl?|w2OsYkgHP6O7Uy1qo*4*3cA7wr*Dkh@qv74xVNy$mwBo_V9 zD>hV2Y`Nlx$!R%+BFD2)W3=WMaTj=t{=sZwz2G-SmgViaRK-jFo96qtDGjX@lOFx{ zo0d&0YmQq$dqZwnepOlBcN=O2@LKI5Hs&$Q2iecAk^c9Wd++^wlgid>KtR9|08r!K z3^h6`xsxWsb9JSs=l1@b+1?^V2TS`WIWi~bPYiwDxP2cw5Hlhn?!)=M2VLW)YjDZX ze+gde|GVs>O5Q>~1eC_3GdT~jz+=!bB;+ds3>$7rbTa6o@zGSb9MA1!RK}Ens*H(061+{}+ zwLjA+I7uODbq%vKEN%U?8|2%%#Yp!|8MKM%{ zF)uVs+_1z2%bac?LevdM$IbAQ-^Ec{Uo#ruLo>ni&xF&00x5BQi&k54KZpM*VyGeW zlS$|+z_<&Zn48DPr5dmfU86R^lNc3{knA5hgoH-T-PZ22xhBCWnU0K7vRbaBCV$Ub zt0B=PKqOgNo&qFDTL}n!D_q%ix}?#n7iPjLLBJgu+(5ImnkIy~Sz()!n5voNQ3fHy zUqbu7SeD}Ry)GvS#3GhPvuO>>@_eSi;7}H)DuF4;L%#xYjk2v|;87RL)E`h&b2B&R zm5eagp_6u$Yzl#f_TAtQ2X@&}GI^Vol1UhkBt$&Do56ZZq=(Y1W`i^qd$ydXc5PlE zvVgvV0z-!@AyH4-GYK6v^?)P(*n=4^Ua+47Z~X@pNQhB36_v#q3rqQGLsV4MUGzO) z5JPfQG%fy5*$^9COw?_>(rAYJf~qRh*9RTr+5(Ts3Jm$+n!|&=+;9qJ=CAYldKCUC z?uD}`S_t&CB?;pmr|z-!{kIgqg0fLxZKd2C74Zpec76}frds#yY10YQ{x`pP|Nx-&gkiFnUx-5fxVa}F%1AFlK!{>jm`0Q=_P^(vs=qfyp*C9^g zxRCa}mEJGqG6eiZcvD&bl0I6!H8_D8+-z!>0?l-uZ2j>dJU5S&Q(4JiPRW+u!6FNMQ8P?oNf8xC)*5@L(NlUOj^&L>#~7zo_8uuMmIoK-`u1ICCO-SjBa= zeO}wmzQ8@MB930PR;m0jn6(1y*ket&zka(o*7(A|-3}zjKSN{|* zQ!zjvx|pkBjqLv7oxTuQczz=zfM~vtos`eEBj?nWI-B_-xyhyE;+?n}c#2gZo`47g z$Lq3obg1Ww)q5(>npL^4C;JtluLMh09BKgXYc$YmQt|)JL6xEg@jpp~-+&0nJ6Rf= zRT}Smky9z#*>b$9re>HmB|D+rb#(@p3pRJ%=f0Y51c#KZPSj~Tb4AaL=HDluGJxpW zo2;)ika_`3L4XJvKZyUhq~ypIXKG#DSL%ZE;?iDw2Z>Oa^VP;~55c@#LNAB+QvxQy zKjp{)X&`gMpn_-R-^Ztvp?~s5WF7~UP`|+QB~uzc&@vpR`OhhbjoSY$iFermOX3HO zwFZ*^ras8UE?`$$TivenBQXpJKW=BX{#g^CfqiR3_;A*o| zo^a2b|4*C${(=Fg;ZVc|K1D0uNPv?oC$Dl zdvt8KS37QkFm{LYFOdN_o8B})ckrh9>tE*6Y;4!@F>03wD797Eq#$rJHNkwD|IuDQ z-&j53$edJDqQ9E=MhKcTTJIsJDLc`0T-vl^+S*48>!<$yWq!JW)O`QgURNcVt$QT%)BOVz zD87e{Jx8Qyu+4hG2}y@HYTbPR|mNTmQU~F!=DW2PM-5{KGil2ZNXM<)Me4Oy~GZ z2JE^d@nvOx3mALX82wYhl13dmp0@5etFnvhiY zPESr)>d^V2H@Cr45*R3wd^*!$b^3I0{rYH>pr*XHkDl9@Ke0VFG^Ww&?mxYEG2bd? zYFbREBA`FrF?Z+h=4OPda%6DqYt+9&92^|{3_%}U*a8@^q(3k`EG)RNSuJESo@-iX zG=O4M?R1)1qjiy$l{G#94d^X}frneI_7Z#WI|BgB$DZV@*8vub6N3-)`32{bBV$Xo zY^Awnxu3q7`Z*1Dq31eEo|igz>F#5!})4aTN|U3jk%GC(J5=WW}~$g zV>o6CriNC7_2f7WITek~GG&ff;{*<;3p65^HtbMWj^t)mN?L}JwszQVeqVq!Ppeo! z@C)Pbum56&eqTs|?%U6giR-WZ6+?kt>1G=p$0fc#;8dkXzJIwl+pKU(V;x>?EV5dz zVqtOex!hRSrfsxg{VdbqYk!$}?e1=%s%+lN`gEOI{O~?Ke>9fS!@xuT>=JU(Wx1Y- z=F7b6-P15aofnPIX=~e=MeKaJiMO$eEu@(3IGd{`(!8Q=^q3E0!E1Yb9)Mp3P?nyW zTliac+u2o{WbxcAa9Nu#B1DCi!N(^j(0lH6c^zYlkf6%(UW5H1N~7P_wz^dabYaJt zEgE;*tw&6!J)FbTv)*Vs?>Et-prsI!H7+ILBs_abtMJ~+@raRJ)z;KxHF=o4?BqJ; zS2Y$fBjGM6HjeFXhym7A`JVTH+z3-B252Bpk@xegl;V^vSZwKlF8z83u5vQ`+L0YSMVX?u}+aWD2KXA{u5$ci298`;SJ^4`>`t z&`~p`_?MPiAbrC|GlqqMvrD5Yg^fg05_feq6*kq8DOe@>Kz>VlzD4J`>H6G1K;ulW z|HAf%0^99c_FQ%D5?^x_MTAnF+odCox$Xxq5ARxDRr34fFu4)zrQ~Z3AyiF zbQ~*|mH9BP96bB88f*QFG&&<5C3(|F3m4-I0%=1;=UxrW_H-f=ft0o=K=S%IU*Z4Lob9m^*LaB{_zQ969#4+E;*F%$;MZEhnn4THLAVOQBe*za% zajEo-JGE_1qSgL0$Eg3Y zgV%7W-ljkM)_T#x%RGc^818vHk9aVylmF`Mes|=4O@gvgFO|%5c`BoYE}iwU9Fd|Y z(_~vV|7)Yg>T+(FNx$fZs>&E-4OLA-<9M&KsNQV#B(wOA8 zbFtc@HA(kg|aMy)yZF zH#&=2kN3S9{R0yhEj3>0W+M$xP?Tve^PliE+wX6@`g3b63dqj1L*gfDLa60W2W|<7 z+MqALkj?=I8#bDLv9fC2PRdLS;U^S~c(U4AN;B4AVRDkHa{4ZyHAb3;NtK8>C9sVb zrLS=G(namnIHq8iXNM>@^jm{YmfFtP*HS!$hLTbdQN-i)Jo?5yZ0dLC;|CY+$0`qv zCUHM&I1aDnP&ggme2I|7n3dbgcBH;d3c1PSP49FE@1`_V)%a zj{}*-O|M5iyyX@rvN8TB+bVjZepA*~?5IBy67fYo9JW8RLPA0g#&<70jxHP~e;ClS zTL5%M(P(P9u-#$DbzIgP%zJmbgt&(ZBD)sJAm8=eX1rE?S32R)n*Az#Z}5F!e1rQq>DOG8SB{*`Pxl>$zkv`lG!R0w zKpGnZX6M<}-->Iw7K+Y7ZE$Cs4Z>%~otaZzE;m{YUN2T!TF_uZ{e$5}p*bO@QZMIQ z56e=*Ey}x(!r5)!mahowu3Z!J>*|c4=f4$nW4ku@Ii4!~U3n6B441O|sig(itfDzX`@uidYgz6?j7eu8(sUokmd*H;pp zEELgb)vF!E%tegu$Bu7HDNA|N?hv%W_QGhfB2Mk+7jzh|RZ2m+)3Y&cenlCK`x!&VQe}z2h@WY=5y0NnN(<5 zcY?+{)JZH69n45gd|tPjrKE67^1h!0-KPb?<4rYqty-_QKtMs7`9PFHV-F1p3@Y2N zHb61f)-oc5@VYwoP(lANJ~q7A-1On6+pDM*v|jr+_QMJS=8QqxW7@!5EO^lM04L7$ z@FeTAmHPAJgRboUh(w?5_WVmeU)rx)A?KxZMcHbQwHpk#yliB6eggXXa0hfiw`sMA zhC@{2_M)-p#9*t2U~}{*HWO`LZyPS~^LHU9yXmB8@Z#M3yz1&UGBGi8bXe4l8;fa2 zuvmc0(G)8+anstxMuSX?>K!R5W&xtfUGleHv9Ik{ac~FY8c)|^di&Va+8u`Hd&9Ff zZTf`7BrfbwqK`|}dA5Le(DqBdY((Sjqy*KZq(x;_HKM6)TUwo#U)R42xLgV^mfI(f zs|+6aI7cW;&uvq*8PCO74bKDKMTM&R84aKCiRXBeH?q7Ijx222+!vzxdYPV|{OLMB z9|jw6G!c(BPk+N4b}KJwD{875RMrM90jnvFm+R5Z8T81>QzS;TBXnPwFx>|U3{zPi z|HiIUTYnQ`c6(!SVAazdh5r@xhxhd5Vxv;6u|iQ=wP`u}9(sjNdo=*?<#K*9AxoWXj9Ym^cZ}o z+d?|6`X-dtKiHG`CIoq^YExDrWD~I$Qa7aY!w=L>-y4qK!oN!Ort{-|$ZdCkR;R{% zexx4r6~S_&3cOwDbkW-TO%E~M;X>moR9-$g@Y;CTN)r_6OiWW6RZ==6*uoI5rjIpz zLDW2B?cZ#k83Tw)zz3yb#saOM6b!9&e~iKFy7k@6h-Bi_7lPma!xlfA^dub^lBSS|clwIv|nK|XJ zUPhp*sJKGS`USkgGc_-dC>Rls{<5gQ85J^lHn(@%O+u9RF)G6&C=%mqpg4jFq-dFT zb3>b;b|nz|3KtEmW?)omgBJo=>bbTWr>q)PDJjjRf!WwRbrEVq(3e z?ft>W=q?#}{F~cOUrGcp8l3W*W*PbikY=X;JD6Y7m;VkX0+?y#uZ@RlnN1~&_6IaH zI_=CBG88Wgwhixd@W)MB@4!t}T@|n4C%$sMPOHg_h-N;Tri&MNQP%QY8uf#L_gPO+tzWed>TU62xgLU!ULQt0LH5Ri6IDDUN zWe!QQIXMI&{axkl9NZ)N!*FnbrKF@niI3Ys{x!?w&|tL@m*!$RwDo}xk_B5i zS5^k_-fm9%eBa}XS(76{{1>yriUP^%gda@h_&e0JU`1(l7++@(6RRk+Cx&iz*n+IS8|&a)Y*!SMMZb(wAkD2!RVXPW=wUOyF=|R`Kf=3Y9OwUwy|)a? zquJsIgL`nd;1b*=Xwaa+3GVI|+=B%O4#C~s-7UDg1$TGbA$gPg-dkH+`(;1vzlx%U zc^+oE` z8ZFOrq|`*iu*Ms$N-IZjAJ-?Hhey9DYd&c2P1YM2WKYFxYTVl|wel?>`Jo3Gd_^07 zzt**163=O7`D5N<#hv-Qklq6tu9i-d7cPXV)@0$XUG(_wSiW>)JC5gh5Ldo6aj9Is z5xsC)Ia=`xI20oG^79>TsR$uN^aYU38vK?n<(;#kjY)Uw(`^weX`Xy;7$=nN2iO6I zLQ*=miw0cs9@dakj#o@V2osM>?oihWyaroaQrvgf`}OX8COlT#Zdl??W)?Bv%q$2ZpZ02tB3cX zV5Z8SucerZGiDtX`b(1j|) zA}KxS|I4mky+-|fe;9t9eH*<;_FiaM9#?)R^hs z$j0yrw_<;;P)$-2JSR;-aYji=iEyyr_jxqO;w*2>*Yf!sVP?gej68uKC36NSLaH;L z!*R7ZoOpI`lHY#q>U6Bp%tiJrt0N~UxZ=Tc)coxDnY+R3B&_TW6J!lKX3Wt*AHd7q z^NLg|SgWxMRs|(=r3D4%Gv(dcaay22F=h-L1`RD0ZBSs9hG!=$B{B=;L|081J{MQ= z(ZhoeSxD^r{w=A6_wNOv5RV*=!JLxph7#waV8FIHJ}excn7 z+8fI}pRE+-Ih%FoadDi0bw6%gM~e`JZubJO0bc$R_{r0c;NSuzHS5-= zX5eIOdYcPKJ`jZcslsW(LHah`Q6LZS{$e*Qhi{|8&Q`fZCC>c~R7kkP@wCZJAkmK7 zY=yI^kiusm-PO&E?mbuo083WXSS}~TB%@*@I|7;I=}U+RfgNH6K!qVd1-+3YmJKvi zR90>NV>XLO#?-DE%q%iMsDGRJ5(*?I7|bXs`}+qt9nI&9j?e+_KgQsjk9Haip_|3* zd@fh5R+Cc-Qn#lQMDL;2LK)R+54%tx%5p`+>3}SuURZ6)2s#RQ0E1d+aOj=3Xf!J* z6cp^ZL)cl$l?jiImI6u)^&-5ON206aBQDR-3mHq`P3Y@Z)g_I_``ul!Y7F+nnj0s+ zA>Wxqej$h#Av24-3vTw@W$E@TJ6JR`cN0# zj?xn-yb?0vf=*`?FA0haKkh>ferL=1K~70rgfC>C;s^Bugx`EiN`C1 zPRoE-4KaYwMxeDFDWhU0LWSf*7L$%K5GO!uKDtGWF72iZ8>zCxEKV3$#)QRuzE(su zYTiQ(5nM1Me??SGtJ5NPYyl3@IX7{5fR6}TJ11*+NP z&_0RUdPNnc+^v~ce6GXsy8kKC-Tm@i1h*71?IH3A@c)a)Cle=rYubN zKqP{u_xBc` zUnHOX4h#X#w>5eNB;@eHi>LAi3FbjAQ_Ge9b3MW|ap!`f*H#=USOm_1K7fr2R_O<(I|7BXxY$EmzSriZvIG_jqw}ZCx3~=ez(7@uzvZf^*n0Z z1r|>sxNpnsqwTyhCc>|Hg1n5^>%sg|uus1q`1iMUBsd4u7OB9;um6f+0YhA!1fVm! zy?rkXfb{?VNqUL2QNRT2{xh^cPs&XJd@p$|sH(q_u-_M}qyX%|DK3ll8xZ;@*a3pV zaS8YqNq1an5dZTa3Ls24DDIPi^Krl}l1+DwH5BT@D zG!HP)+?LnIznS^(EZW3h;w+JAlp?>D_Rsi9t@mCAdg;Iq_xBOMFQi{0kx}5njK7xq zZ#>TX|6xginUPWk1J=`jt(?otK;7|2LI3ALQ9xAh|6>xUNXv!&qS$C1v_kW9i{8DX zh20LmlS+e=mHO4huNd>oa(b`7ERH%2Jis)@7vQEjGNs9^5||{P7de2iYm12LtAED+ zQlxky_zx+(`6*;sbbLg#;a`U(Fp-Vglsy1M&1HWffYakBkbm%7H2cET6&AQH#q^Yw z@o}&h{K{Tlk?@MT@uU~XdHL&KKgJ9%p<8%XX5xR{1io<747~Y4(?=4>JNQdG;9^w! z#aG2LHT+fV_MeYVTmiD)HiIYp76^a2Gp!vOfWsIj2Lu0y@vnguICq6%`sH-|hkpFN z$g8e@2L$xrXghRFO~Vq&JOwXrsp9a;TE36G(UnO9`N(^V8p0U+8*|GC9yXz17Q6W} z3vWI@py&Zb=KUioMb{=cdii)Kw<>YUf zp~CzovRyu^(K;=T*Ouq*X;RtCb^1BXCP((YpY zV_w92F2^Gz-AY>ZI`j1&TE_Arb!iGb(h)-zl_Cr<7jsE+V=-xI93E5S&!6M3ESdy= zrQHG%^*1jO^}GsIg%={;@PLmp$|)(a-8K?#4QrS>CUkiN#{dDvdpUqs77|j4qr(>o z#devRiWBh*5SZQ8Tdk|9vHTH@4S?W|BPEXp*_)1{_zl)Y)2n5Dqu5JZiypn9+sS;6 z<5^U8M)c4jUv1~`>2l$kxyz=Zdm-GJ=zPb_t>^JGd*Sz{G9^@0@?>t0sgNgL!dZX| zBl5yZdc|XgFN5QN09U8OxvGkfTo_WTP^R(&dD;EcI z)fGjPn#y{mRbq%=+jrV#8NW)}&8@Y>50Pi~ua-ugMg3dk_^9owAsin4FxQV=If#fIs=Kcp#w& zAdv~b#0I|%IxQcC0Y^hbG;yN~KwTY>p%C}(w--w9?+W{Y>bi=`o(c=md7xTs@fpZs z)5~?Vc6B_IPG+>@?eTFq9J}38E0vJ8gh2#X0}8Bs5g#wN7M{@XvC;AH7OCs~r@s&x zLDn+AdpB#rr2XwvD+mushu~vx7OP?j4G=*9sM*$!1Z+d;FDM4Tg)8 zD}{5rQS@O>PutrCo(hCACk}mFYC%$`rdV54@cs#5KwH1C_5-N)>C5z2m66-JT&nxx z$F#ffu&UdIKV}ZVV{JP08(_XWS=A~Md2gV27};HDYj^erpaD8Q{ue)!d~6_)1|W+L z^-0!p_NE8v0>{W$D@NC-(Vcz^Ebo^NE34By62J0vT2s7wUA-r4CiPuD zW6rL|s((Iprd4C@z1M_23jt&W(m3Et>!Mj9CgoVFY0R7xIatuHwa$Yc$i~y{zXXJxCo&d@6A>MX$}fQE?yX_`JNp zx~`kaDs!4RbtNSjRW<9AWTz!L1$8C>BZ9aMQ?10H)vCzR2-raGg#6mLn&b_3k8hT_ zk@>to6&)U<4gHHu{AnnF3!66mZs1q4Pix96NIPdO`5;F90`@I2GHYdqG#YBI*Y{kF zU=zx#UjWRMfTywLVQ=F3afRT5LZ_@@0RaC6YLAe)x<#$K&N?eQ8(ehaKTOqO?&Fm=nF|Y_3N85#i zj&3;}rTn$0E6b1jsL8p#WB2)bgM_EC+ODJ42FDgL=H27QK&l4|j8#MQ_27+#3eKw# zus}UKJkfrcuz=q*0s(e|q?aN`h{THoZjJJ~2-F5om?=Ej=k%`BVM`p_m;2^~G!Q5( zqI;>Q`h*1&tw*9a0yJtl_FNrKjg$5~Y~F5gvIuZenx0=`HX4+XTH3b4tjY|5O6Y^5 ziGj+zycsz8-Sp-S1E9NL23?Huf3Y4r^5`;LWJj~CZ)25_&nphB9j5m7DwE3b@z>xB zJ_}VA_~`hX`^w&=Y+^bo@EB-`9H-+>d>+LTaW#&OJU~vmXAj6{E?#{*F2w^P7Ejw+}nA%iPH?n`d zOR=ws^hW%b{yKRkd4o?PaC!-53 zH=)nJO$2R*QSUXy-`6K*(Asj%3#(;~D<1omqq2nqYh?@gn+BWtuH)=hD{3EA53MJ| zusLr^TrGjXKa5MV^bFic>)B>jzCdhRN5{{Bl6VHw$x-b@i-8`X1mkXVi?*>$Qx(u@ z^Ht(s^fI*qgB0eCkhNe$Jt~boLerzkepxWa-(L)rAs2&Ri7Kn%Z(dn-7_+Ha5Ru2N zbvlt_pW^>yse`v{>@;zj9U}Lg(6oD`oq2aR>prw@aDwHcFlZ=0phR0hOfmS*SAEJl7;m@7J$Up%D;h$Dmv@-sa0-%Kz08 z4CgIil!aDr8MD9a9F}(=*pHjm4Q6J^6+-a~dDA>>YJ(BT0^;K0zF%HH?eo++7|)0x zpl~;x%_6XCPUJ`}e1xk}?P)H%3GLOZb3!19)->J?>1Vs8LimJRw*!sbM7} z3r&-I@73e|WpE($1n-yDTyGEb8v;PJKN|j^u-m5nAhYDeaQv-v&3u8JlZNJC{4Krd z81WkAz_)kgY+D6p^aOU(42eTH7&xFHp2?#9=h43;-iM8YA#5eGDJ>!DEhP+dJ2=KmD^PxQS*4C0&3rL`?NqeJpre!N_ zPxs{V`9T!PT3K<^-``{iuw}b~gZYRt_kQ_Vcf>%h-5s$w&&$?ALIPh^(0>2%a=CS| zb)%VI?T!RT z?k<+oaW?H!=~dgLS=iJ}AFw(yBwp`M-`SO#?ZFehBM;gvTx@{ z<2iiXJi)9XC&@{17+2IMM}!tP&-Qm?2SeS@t94sC!A-63VLbiSk$P7FF?hHR3W2T3 zqQdyj)SP!PAw0{i>sXnj;iyMawspX9g3b?qml>5gIj74Qy`P4O^n00f0Nv2GD*~h% z<*J2_5&#DA9g(bCKN&0ElL}u$-fuYVMaNbVwRK<0Bo?IolR-sl{R1X`D0dMY=hqgd zAP3eM^<3vynaPV1dkc~%HFDfeFyW=|xsfV~!F44gPQdr$?LiQabF!E*)2%&@Ma^uz z-F|hR!Y{86Xsg1-;0din4Qx#gDVUv_%u@etC+no1)YASPiX64r&oWK9LCzcAl$=Z( zaT}g7UphNP_xR=wt)>Gun~4Gz@|Te^g*9zFyHe@*@=i!(!YS9wKZs#_AiYT&EpuQUtvrkBaK;sJB37M&8i(;GTZpJl`~kryA>qZq z`Pa`R6v(i5K4S2s!2h^9Z(bw`w4vc|AKyC1_ISUq~-s1JX{Qy0ufyQ;~?X_G#!L&AC&x``+L5?{kt?V z{D0KUpMfF>0vgm;+mQT!?q3Q3_uXZ)UjLUP2&Bvbodk0P7?@H2=l+;<1=%M+sI`~J7bmnQJy2?BBXjpuSF0YJ@Ax6Ru?6_;JYm zmbpJ59|SgM2I`95x+xXOfWSElxT9r5XIyB4!B9o>gPLSrx6HNNKORd~&=66!^a?^7 zZM;Oct{3e*nNNm2tw=zHyr7Xli21<-Um?E%6o(i%Fpdv<{Z|KWPv{JlhW>Cbyg za6#FUOBAwyCcah3f2f|{&)J=n1zuL72zn^o77gw1=m&E6UViQ|9re!;@MB)V#E0GK z1J!DPC6&g>fPb?lH}QMGe@4$QO2QAClgabZ6!B#ge{R1*20T!)Hv828390{?D}Lz9 zQ?G>bdH(h7mkEHOz5$u=CZuBx{rAdvLEE!{l4^6!qh??IYr%jQc>@*~xFnQE-6YGO z5&wST=oqvZ*C*e+mlsqqgU-@ox^kbY@xb6ax|teTMsojsZ(Fs17x-ZHX;PmAvGpN} zmod?t?IADkS<*iP;CFqQTr&^yS-{IE=5uo7u~n&KKIDH3aDXLeofsovs1Eq&yKOOK z{C3(UG*$(n&<}Q?i%2wTQc_EHBg3KLxUBXkeu{-N=Rr-3A>Fz_g0EJqL04J-DNlgM z$#O@J2rbHOqxnNDe;PYhO5&qwc%A?Wqzr&KH%^&uPCVChKi3KMoQ6h`*7v%}KScvC(l4Eh$bpHd#LDd$eJ>%Hc4vZ@)fo zY-~z|b4^;7+pT)&+*=^DEc6x7NfMA1=5;rj0F3 znHJ2q7h6CT<~mY8XF@t+Dawhph27?UndhvBk}a(pNMb-VL%uZk@a6 zc@YKw;UvH^K>5vgT(3c(U*VlLyM6(`*F8`c@UUs0bIVHeA7$X6qNWBK#-Iw22l`h# zga*7O{~7DP*EDAQuI`!POEio{EXaGv57|ReE1w_rdMEeBQ-)Jl=}H3+;))w?k*08*z?oMvPv|~*swBOMZh8Nh=At55T z%1%~R*YPQ1FAWp(rrf9k6Dpk{fTVz)$X5{`$ACK9#U^Jp(;tk*O8fi!%*K6bb(-aJ z`I8YigE<7;5h;!j2XlfUQ#dn?+oS1V32(g5%0KySBdRec(&DmP(9%-#xUgvC&LCpX zwo2&Voalg&q1-w3BU2}%{$=WIJiIX6BGm=V#Tuso_KwsE>!RL?}97NqN7=$v05h6{Wb~;Tzp%0*8+}}`Zi_Js;eDB zc%SbYk3I5B3Rrh9iGx&VV(RM)tePEu0=Y5Z?j1^0sL6OG(3R* z>VO&yp&JDM;mb%ul#}DIgw@efL>}`ltqY0VOwl~CeBg1j(~m~Nrc5R3RnlX(Mt9;3 ztiSTpS`qr2S3UJ?z8vw17RsnG3nNeFxSTZ5SJlW+Ki~^w=jazH?m_a9@K%{tkmiic zM%`wn1ohKHc8TMLeMd}X%*uW@a^}kP^pC{~JzXM1eADWhl`cz&d_`VJyZD%yHg<7T z`ohi--Q9`M4DxS8YJ z_OAIhLaR9BUZaG|nW#OwY#3; z=J(S){=#Ik^nE7Px*^z^%X#$lG#>U_o%?JhDwT%)8CFmDGt6>CQAUOv#j4$oS>D84 zjx_FvR*8uh7%_5JT}AhvYR~7#`_Gl8hvCeIyvD4FeuCvgx|Q+67E*RA%}%^eU5`L{ z@bO;t^Q~Ry86O+X<7(!NFQ_{j>=3<#=g9Ez`$IQZ8$qbQ9Ka8L07#tKkE1Zi#>e0{ zM2a@eu61?BlH~8j#Z97W^7FD-_7!TRClC4BFJcd}7jtL1*0o%`_UuJXti7EbL0S=9Z83k%?v4wro}@>p|#A%_f2qBpup-07IGlMoRoOdTUf95w4# zBVvJStGhCIfttII`+X9Jvq7B!Hz39!#~i2HbY6qK^vNJu_>IoI{{mExM{Pq6U27T# zJ$-9M6N;0aEu5bq43|3&4sNM8g#aDOU$$2E1?-+ef#f|>&}Ui@zb85zsS(ZcMn{6- zAuctIk1=-Sh2o`%=jVaPgd+%ysD2JdlcieW@(K0A5ZSYMVmBZq@|T{hD(gO0TmzGPAcjjcqqen{M$kzv|>#%eZGWMw)!HDeNL$b4XO zDH0)RS&fsP@|`D*{$I)`Nd_>SZr|+$LswaWj$&Y0prg+zlEtagZn+c_fOVN;6o$tc zwR+gh9ulm#jDnynWHw4xrB@ZXtbeg^zU_|hM;WIQD^}7WhwlkjRGy)K*eE1yQJtUE z-r zGypT!-t`*l+vZv{N=Y;<1helw#x#X*^!NFO0D{ldjBAuaGV0eQJ2-Ntlgl2wu8!nc zWU2g!h-Y?&JHG$2HH%@sd6`K0l?3s-lD|^)s&4w}(ko&QH_+Fo!SKA$@jy%&yDHr- zXL*kn2)j{lb9OU=>7k~i+Ul{SL{kAT%wTvRMoGwm;3Q*YcsOs_%v^d69lEj;GG1qxFL9q9BLwY3y$ba-u4HcI zFeC&-n##cV51&r){GEmJAhK_~lmQQg%1@5eJOkzx(n%Z-`(&vmlOI|dzUpCvn!8Z> z-mIbSME%!{Mn{s0gGO1AjYq)W@QwCnC=@(^JrskQ-JQxKNO6Uz@>$%`TI25dT-rl} z>ktVbBPT;(s5w|9E0tcKTJ98{_35k^IyQp{As3f)3nPS#RVdMys7Y*Uz4_W88- zxjeAjT!1+wfe=?o^?lAY+!H9hmP zxFI~;g`rN=C7L1B{qiZEoYh3y8b9}dj0Oh*o6PplJF;iYbBh8GbG$(^ctkf*I| zBK9`yY0MnezQd_ckFLEUqNKU{SvgsJ=NEnkIYE#l+1)a zZ@0L+V5HD$)Ev}4_@J;O);7#He3j2FG{a%wUNK}*E0TDPEl_N^H0i(dKf{Tf}o6^TO0lG-ro)hP>~5B zt9_Gp=JNAP`uWuK8L+;!PbvE5KVyWjvR~5}?%5wL_ICq@@XehspUtd*=N5d4cP=WPVpX!L31W-RG|ItA#4h@!*=Zs==Fp?DMdOOWl85?vn2To&#EjwEOtY4!Kpq#7|! zqQZq%TS={H9~CaN*dl^8P(U7VJ&vGirr~n5A|ZYVy&y1k9pJ`oJ?N0qQ-91*V^Q#f zJ%1^${6aI)l!&)4c8Yfhk}*9fB1V@VpYHP^DGq02u1yGx zw8+tN$(W#dD3uv933~+7#XMajWH}_pcyc0q^7d!b`Vzu1P4^vgW#3EpH@2cuw#KTOy@-J!0|1vXU3h?;>qzXqL-^E`eMEYxw2&M6$8f(4pn`fpv18DBO z@AnGQNrwh2mfX^H=;yv4DC-|h!tYA<0)y~kpM?P5D-?J&FKor2N~4sZKocci`UOsb z;_v1^Q3f}DJ*YL7?h34idH{Hl(%V*S^2;{$0GZV9lE1;O!&JZ!x_bWF_x=w7|19Ho zm{C#>7xe#~1Hgd*Ga3@0bAI>-F8d2J!C<*S{r}_ZUVcb_Xt(nU9E0`~i`jJ1>ro_A zP+;+&Nxa+I+JJuT&tuT&RwLkfP&qt&_%CGr54m4B_#9YBPDCh;|Go`EJeu-i`i#f~ zxcv`UC$#zwlm%+*mmB_To$^QAC@9;8Da!I8OFm{eS5aIk7V~5 zkto-uf0_y7oX@ob7~@G4K>eOXy%hd{aj%d$!o8Fh6f`wKPIW6)H8eDun|VeW53SQ@ z_TN$$NRl&mg-sbA;UEvMup~$Vsiq<%;}%MZ!jwoKNOcnwa%8Ek`!<>M8)X^Z*f~g9 zymtrl3+UE8Jv*~^aInccT+E*`EiNn!7}a3>(+i+9zxX}M-r4oJgPmPBr&W4R4kT;G zTS6@>dYN#$?qNPNV`D#L1%3Yt6c}!QaIKy3TQa~P288N+z&PGq6I1#RXG&Ymh7oX= zuddV6V5Co(9!+vyOvtRXeuK9qW?Y}#mEnELC>JoQp6^20->V)FQV*9>RlSYVa&zKh z9?@k}69#>thzU+tQ=oP*&P`h*f$Vh*Kecx9M|P!v@$+lfRk)8KfC{yV15POi3{3*ri>w29$XWxX zY;MmC7v*&hHVxJ(nY1()m(lP+8^5Q}Ll{2VC^}cN$HJBSLM?t3?3j+O;@j_{ZbTH8 z_=E%ZBkd8KHkv+Jvcpf+8bSZuPYL}VbH`J9iNun`%&^0 zN+%rm@Yrn55W-@ZxEF1WwGs_Kl&?fwW9Ro&>85ZvAAH5WUVI*{Y61J1Ehpe^A!RAu|9uKK#qGeqJ0$M=kYS@#5;SA1 zgjWh-p+b&&t)sk+4vb%b-V{fG3RkQnF@3HWbJe(nPrO~?@~o1I)9V#> z?)h!2c0)_0qthz*bVVA$WLXCDCiTLv_Tu1yUX{?Yh+ixFA~pVq)){;dXppW`S%{)d z!3YH8me0>A?sZ=L0)Znky8(VkA;P^A%p*y|OsX_A=)|CpgiYU8nPAclik;rybHVZB%Mh>DbtawlpdXG2^$00?AAwgr>vNOBex1kN140!s9*-i&AmDLL;cWKWHqJUn-xg^_Mw9S?Ny0>118C8&z&{b*W^}& z9C>lwSu=@610I^rKWD|^S=P__-5Egq*VG|1C3BwapK9YXb!X7?(B{t0VXqIlX`%*z zeybumb3&AzRPlzWCphlZO5~kF8Iv1PP>6V65ZzSuy{m!7Kv`XAY^##`ouv4A^o_7T zEk<7yqcosAd-6Z`bNn5D*efp9G?)#!{;tFxf5;oQgu#VcOHm-%f;0`B!J{av%55YC z++V??6QR@fD%i03jX9E%9U|lek-0WzkCDy#0EcjwosbP8;`K=D4P}cm%T@*+cSO0~ zHgD!wpeBB-040A$K%AM|ib}bHy1YiE-nIvt$wz^~r%1Bac=)Zd(N#vhFVsYO@%FYz z41L}a<`@eW6P9Q8${2A<1mYi<^`zmQPCjheD_NSFHYgAu@x77GkFDSyRMf0VwhE>f zNqy1nN`QE#dn?T(@{dYhbs=vPK6Y~gSLj@AcW-dqr$G)vMHwGXp#{e>hSBUfm@kJy zJ@WN^fJFSNOq~_@ z%eXlT3}{0GorhQ4xi>D2Ixv+8`|ONn=3lYDLYtcHvlp)f*WbVY(uIhJg-8Vf0@gar z!;;gY(vcOU2OP>YnKzY_l|uQp)N9*~AhSI_GFnMaeYsM3u&Obq!8F?ZDlF6Au4d;e zAkK1MLMUiO`qBSr%{B)KuL(F}Nz3-=7&Xg8N)(m1Kg{Ih=NlJAxX~6SMkbnzp*8^C;VVu#;Dk09secus=^Sp%$II(X}0f)_WK=Tb&9chyka#t}0rM)C&Q<(;VHm{gvWa zvr&;casLp#w4|7z2PK?2XT8IOV{$*r&N2Q>E@l*I5{-^eP!!>fAB_51rFdbY^*WYV zVx_Y`)N4|lOGppAPlkrmV^!-dyvsm83Z{gwv`LWrHCv2^n4dquyfB={3&ZirSBU*# zxH};5#8)OG1}52Ft5W`9*1}1W2?xQ5X_!EYyI;^f(W3u)po=)Wu+Xrr?X$EX1B{o+ z`Hrm#?=0(6r=-ht@mQqGGJAtm*$P}m9;+#$lhdNFsUYJ<&%8Dz%VP>R3(iNq%%ip^ zA^S1@6H==R=QMV!|f_;FzQUTr8ipI|acVNF+y36`m6er@Q;RG3u4I&TG8^(*a$~nh=6g!oLw!qWx)0gp z_{Dh^3RGQsR#+hd#*3r$5upoKIrW<4akAfO-$MG@)*s{~_K3$+xI?`*(ogaR?~of7 z50LAEb^aWOq~)ol1UuCA?z8DFvQ_g{hg7u2C5h6s>3S^NN^sPYdns$ zTIs@}AzI+D9o5FIaGwq)_#qq~$N1E6_BhnKG_c`(G{>;F;8f<;s^?9f-12!Oc_B@U zJ{6I$U-9hhoT9|Yu-Lx9Q}!uoUFmY0Z~9xp&$2G6o`7Ef=arTDRyySlb|)2Oui35XtV93 zkkQW}CFHJqw$^aZ438Tq`}qDd@6L%fv%+j$W$aFBEm*cs^!W^j2&|>9YS~nwM`^y5 zLSZzIt3$6n4!Ji6kxye%pia&m5Ix=BmY3H^{O$^9CiDIq#xD;i_b zJfU+$ANsWt)jM6VF>oL)_0(o*Oxc(7ov?6rW6cqhs(CsQNI6eqWs+ng!)`~#&k*i- zm>1;C8ff~(Fq|b(^T(`FmjPEn8*yQsr9shzPrKZq#pZFWOD#1rQQbV-Y$d}N?@fyG zcytvNb?pA3T?7So%4e(hUi=>9y_hyh|1bo#q-`6`q=ag>k4(#alQVnHkdUZj0)q~V z5@;JG>!Od(sseQC%$Ez-oE1nYxiCDfs-4`(NZWcjuFF{5?{>uW1>V!T6pNh4p!eoS z$hJ;SxEy(~C7sgp9V}Z3Us$qN+3IV2y%#U*{x~^2}c4(g*jEWzt0cjkS^%z4>T5jSkt@>&*wCKA0F{ zemPj2F09Ltm9>vh5X3fvDAWCmaIpJ%EUI&;|2b z4^^{jXdc#r@^w)`s#EF8F~dIBlv;W$LutVp!kbh(I;5CC z1;m(vW>o$V&Jc>#2(u~OOH-TaqB$#$y=X;(({h1o5U+A>tkRKq-hCtFaTPk05%a~J ztuz>{Lx%MS4Tkm!n85!=$acWmsj8hSId1Jvem8r6F?0V25CQuoWGo91wKmRO+~2-qqOA{rrnH+wdjA-qNZOL zS8Al{T-RP7<@nwevp(lC;I_BEVv6lO+|dSM9XtG`AHWjw=10^}@m80qX3cg2c?Xm{ z?zgvMIs*{pC^u-8IHWon#LQvK#UQbLJ+N|;yJR`!bHDD7&&kjsUmPthjr2~Kvz=#m zp+u+?%LR9-A)?D^hM}HvGkhD zh~rf@*0z4`u7TV;ep_TH+j^_uU9RJ_=pd2$bUNEgma$(?osrSleE!rLEHR*$(WukR z6fJ0ohRbUkpBU$)r?;M#4rUEQ1V&|dPAl(nv8Sl4#%`^xMS8|b($Tr)ezFyG0ndSf zh2~}7viEItw7#^JSl9Ijv_2E|_%2}AtimE`%a4}uvnoLRV`YQnqNr3Ej}@s_-cZNV zYw^0@+2<;W_B?-oMW}Hy)~YwmHgT99sORCDp=YDd-=N3V+sIKHXBx9zI)Tr8&Uw0< z7xn}

    h!;VT4LaMFj&7zhTxc{_$f(r5WkHa>OeHqQ>BpeA7R&2>;uw49_DD3W^(% znylK>0TVkAQLKkI3;}4Gh8)2N0g2nv3k=;$LU;9&v{V)iUE576Pa)VQG175>X~Pi? zJkE|lt>PbmOlZ}G^i}9s&ftycuKCv)2%Ie|lc!4Ry2WDiClWZyg@VkS7}f^Bcxb~0 zDBFXr&`bg3YA80L_ zA7znX;@>_)=ZOG(TEYmrH8?o$dbE}tDb2y2 z0#fFxe;lh!V{hj*lz$Al)>_Yq_&^D552;tNiw4T(+a8@Xn20emIoi07_T$A#^bQkh zo1WBvc9L&{sKd{9m{C(JVrwt2(rYbJan<&W)GX~h{t22cMuXDO(D3f;hG)Gu^y=)Y z2~_fP_J#_w9FVlndwWxvyl~y!eXV}7>xxtT9dRGO(qZzv;jV}8dGTTY$7MD|H+iNi zC6Yu|hUFM?IFH9jUqt7;b0MbtW#3iY1` zF)}qNP(lDKZd8HyygV-8_2Idv)$q&VqLo4FSgakL8zd%bY!oaks>-x0*5|ZQ3$<^}2`!3>^4yMYJs-<03l$0&GPyeqkzxH%f@mmejVF2z=?4>l11#y&JlA2l zuWF}G=@m8HJI4v1>}vAL8+tyIGrP{75D^pY>v^-O)YxwZYfZnMBp@g$=(8kRYH;W* zb}4r39oFcT*_05<(DHn2_I&0=aZH-AI(3V7;O1bH={rPeS zJ;aiSciP_0>GDJ&HV)l!4+V)}8RB(Sut4(K_*69vf>+*M2z|>+^LpC+k_JhfbP|i> z#ZEf{!n{d;=x_)NeGK12&~p_?MZLVAA$NCspT<5y#~nwj zMG-;G4dFcO`dr=q9+JVtuu^9*o}Hj_mO2*t?rOKcYQSL1(Cx6aqd{&#atIM&mBXg$YC{d)^iQy$HcNQ#r3lIr_Mb3{4ZqM~t`qpEi z!Jt;9glWIp{l16y{$a5F{e5Bd2k;pB=^1*}rk~s01Gr?QGDBPn;E`44fb!Z@zKXxvKu?y3nmtysRyfyQbe=@OT=w-euj7?!?Yk|>%BQ( zg&Bs;oJ5%C+^bPbbMHL$UD0eg;krSP+*hhG$}uqvU84x2_nlid=2q2qeHmyoxy3P!(5E+!fahR7XIm6mTg=Rn@iZzNY1Zp9JIlS*V&;iM^*@21f zR*7R?=tnor`P?Vg70sF$Dp#&#z!iN#rQi$n{|Uy;BZK&jULZAMIRvyxmScn0nh#mUfau74?pX2d~-b>$Y_mShh9CRy*hTA(b1! z)A%V~{Wr>p5B|Cl3PhWpprVkxh(CvIfWkq1oSx z3-dQc`s7<%;&uu6_^YaQ?25$CIa z5gyTptrrRwLo7BES^t?)4;-_HSIn8RP<%>pBSag9Z9Tlw0dxE+wOzAJ(Gakmq=^AL z=s&1(6&xsZ@pt>Xjp%iMVFU6glfKo@qL)8+e#SGB9JXVp4r?hX&k*psWmVNpl9vVf z6I=1E0B52(yCzfE0u9$4k7h}t23#)!+sEaAqt*pe!#|cfonK%v+4Fm~(n}IeN=SYf z80zPgtwFHeW?D1@nuF7B4O& z6Nh}}*2Gq-OkAZLSc6@iUBhuW6^fPS-w!eyRfH4rWl!z7b~$!3y>bgR7ISfd9JjzQ zD_!Q%)g!@A{JNa$e*b#YU|W^uCPFhymP#}fJ8ZKR?;Oy9<_itDZ}GvQZuRt&-4{O! z3vYhmtEEEw@$Fk12-v+?HcCs+_wNlnjn`93GrBbmD#Ug+PQn3^rKNG1Qumy}X}wvDXSwZ%B%6+V3m`uM4K*+V|6hKk6W2UEp&8w~R55D!DQ z)|^kr9l2jLR0PZvIjF0|k`9$3T$y@!=rOObaHZV6R@w@Pw)OCrKKNu*MQ7%jUF@t8 z5eZNlFyKtcg&SoVWxNH8T2{o7(zmZo;mIK4;7rT_rkKK;9J1U*^#0!e%NHO~>0q{8 zc_D3LSJFDOsmOF-n#VSQg_J}4q7IqCmWhXilT(T~*2_`WIxSO(l9rb5GlRI=$}1fN zE0WBHs{?G`bS*bKuK|4~LT*Q<#~bm?F-nXeA_YkJy@lepxYmJSRySKi3qyu#&d#hX za1C0H2eY1M8`$jZmy?MU>7R9UPQ4!kE?syCR23B1?+1LV&X5MOCBWb`n~uE z%|^p|60yHs?bSV0eifyUnn7Ziu%Mj|>^G9SziA=idHsVuM zn&v&*!4Cs9?EHfN*5gTPCM)@L@;zgD#Fh2!?AQgwd21i9yo!<%gtD+;ggTMN?LfoA zagy6e^TdHQlAPzw5oo0_i`#{^P!)Adq?N7Z@SM}ir#u@6Y*th4k-DS!K9JWCx59SY zP3}K$u#2UQ4AxqNvXz8|iagE|L`A}NLxSc`&#r9ds90uadNW%zovzH+_U%^O_Y$x* z8vKIm$CTU}Z-_Zc3yZbx&I>xBCP?M>0~tHxp6hgP&m2M#xM;Th`*+gQJrLiyj2zZV zm9xt(WU1Ai6uLK}3~ym1N{P_02k@CsM=O(0u z$sR|~_e{iksSu@^`)we1$-&IRVksEi%tNERrYfsPmqn+Wt8a3&%&VV0wDCTLTU}#r zd;vIDd#OpMH9Q>Xi46xX>8!f>5Z?NKwe=RjaWt{IsF|5#W@cuJnPNL;T(gsynVFfH z9osQ8bIgn}Gcz;3b@HEcZ{2!ZQ(L>UQ`6IGNh`IaPZji)RIJG-bx0L{S72a#RJimj)U6IEz*8NxgDzSpLR`hy$1 z$ayKXH`qQ-*X!wJ)n>UO#1rZ0Fk^xlevS|kc<9rnuKAUq94|5$d2Mh8hz4fG@kYE< zUm7$i`ebF)On;AGd>RCwP)SY6NvBYp|c0975O6YXy|2-IAnfhHy;oP1rSy65^)uG)Rh@l zDtmu^3IB2}|Jg!IP0g|{1P||4jT5K%k@*vYYNDfCe0{YDd@{-vshij1E*Gcu%IVhL zlt5vrH$eEauCCT_Xu0-%paWc&TQ?5oobuC(bEDNJa4&msSdZrGFK^vugMo&2qk+h# z*YRQ$aol=1md5r8=u0>c6y-a`-&aBv_VAh6($n&Wshq&b{fm85-#!pFh1Pc7b1Lwbrs;~b2{PipRJThE!(f;)Pj%uiqD5A=1*nBYinpJd2Y=fQntT|pP2}ow$&jz zE(KA(PGcUIU}21y)Qo3>$%fO?l^&NKNIrpLRV59U(H{@CvUCSdr}F8&J~ z`8tV!_+o(l=#AH4`Ba_UnA}>L7CBM_pUBz1!?u&b;PT*GgjX+7Aa{3>?>y%5gc?!K z8=o%p?*GO6aW^^v{nNVFwY`m0OlK!iL=`tCYW^P!)=yb3Pr2Axo?wQ%W8UrWyJoUK z?fQs1t^4AD_I%&#ak=`si~HF?w#VzD9}zR#)~&PNC+{AqmxQykGsAjT^*~M%V;>cI zAo$~RP&2w(`X9g~Ha5H9`%#(t{^<8vPKKwO=Sr+s<3!oyceB%WJbuN!@T;qtN(BrC%$TJR!?*Gu6}Wp{jbIg!3Q}`B52oE=T)v-zh-ie=?>^0&tUeIQ;jJ{o>V zOMfKZ?FsxuhK$hV?Y+Laij0e%vXm4d43?AnaM5?)1&5~AJ&MBtd-@2Th0Zv zjFc|viz!&8Y_bAgV`b0Mil?Dug6drF?IF+mj58P*_ziDA(I*@Z_P)6`X?PE6TKd%Y zbL*N~*_pvg(N6>0h~jZ zG}wNL6BrJ9f}+>2HaQk7mFIS`iC$zH$K^70`TTjip;}IcVAX0()ew#gSX-;0=)3UX<+sh?%if4!xZ-~zfOqF*i zFdq`Qj|y}|CrMy0lrWxm-p-MfL0DD_0@Mb%-u)+Vk8j~pcix_@JP#9#UOn1Tlf(l8 zNA(}h#=crHR>;<`}TEI!e06tg%iAp zLZA*5YED8v8CcYU-%IMxtJ}++R!ZhW-^b8}#o-AVvxYzmV=PjO?b&6k05_4|h-NHg zA+L#F)!^cyww_Xu54?w5g?VsbZDeMAN{siC! z(Q7`LtUqKi<6&&v&$ZeYt#Sad9`h}0D)e8ep!m8Guy_D_C~H|nv4;Fue1Q1qa7ZwE z6^34!?gL-u<;h2%EoS#^#C&Y}3~c;^`4ial)E+7eVDXyg?JCuI8y6M(q%0XW9mn@l z#ixPnR_k<+bx9)6ia(qrd~{J1?TX2&fLUB9#K`n_9=32G(3yTVWOnOgYb9jKq2B*UL+r4K~jdx*yf=Zle>x^+NT{&z@rCP~-mXdx6*4{1hjk zl87LBoVlv(d^GU7Kbt!Fv}BRlBii|Jlpjpk{G~uec){<&A9-ECp%)OL#_z~V_bi8G zTGP~AvYDhq(sN9nEc1soG&p=|CX)!fUk~ao8I-`M?V1y`(7If^b@4l@n547j#Ur$l z1fsb3_xhu^=W~y}W=VlnT?^77+1mZz(oMtIy(g5-au3Tb*@is3`~cw>aO9QZm=FlG z$Kl;h66|^Qq)a5z4RYY-Nd`pH=)dwH;Z|ZFbA(L{?Xo#jHUPM6KB|?it?}~NW#iALz7q? zBJ9rT?oi=UcmNE=Ce*Z&V{O+4Jy`)o>{#Undk_cLJ9!7;?_k93vqhXTm*0}Nt! zzfk0bagTumQ7S(_^8=?C5x5k*bnx-Z~AIX zN&%pdk8Y_nrH=}uLR01`6Qq=PDsX)xmHa;%4vz2~B;%qC7oi?TiLl}iM^iKWgX6d? zmmc?tmhJ4|CV|~;zOjG$#70SpL&{Gfq@Ug00|mx`-m*fHJPZWjA!eGH`O2cu z3;Yv?ao1SV;KDD3Og{sBz1+LWDnMi@)Dqec_UrK(s*flpuIQdhoAG}cvlXnMV%9KK zDC(gFacwVoZEb!R7tG|O7vE>t-sV!vR^n>DA4F`Rcw2l8naqZNfsliNh1k^Pg??}~ z+ucb{P&vAV8vS90Nr1EO)gtUOyUzPsM)(Y8B;@e3DUMBHEz9zTsw#5MpvX8sp)!#bcF$G= z@*1_vc0PQ$caV3U_xs<@`tK3aqS~CNYYnc>Hs+IEptf8Jgk0Ze71fjXcKc4Sw6Yg0 z-ax&O1zqMX8|4+Gq+Y%aRl463up9-h(*qGo{0)Agk;lWdQ^n-lU4|HdZ!mQ3VV=8Y zKMP02~JTE_b;j7I!U#NB&Z}9sjMjB{-)rOi)l|CSc=D5Nm z&dJG3o?ykWdN49euGNpDl1n-g!@3#lB+e3 z?2QOShUkaSRTZc7&*w51g5YoG2c{(o#OnehG|wYp>k?78VTzg3@Pf13_>Kt88VR97 zFiXu%GlT_;+5zk3i-U~5h{uAL5-B^vZQ&eFiFSfd7UQFk5%knM;GY?LcSxnMgzRv0 zc*w%o@E(-1Lcs(NOd0`^npkjCo0A!d?|vdl;l)Vg#K?S`j47bO;C$`12x;eyW!XS2+EZ4o=%+B$uBZ$^prt>TI z=ciLjsKH1D_(EQ+kAqWwFI#bOVL+H^p^J-~#ffpB>Fpmp>oBc$asy>4KpZv0%!3!d zW=)0%US4;mNBB!ftL&Z0n}hZ+k)q~mG8RW$r+Af5VKgUy?>*)<#xr!ffO z=y>xmila45mX7nSIy3UL=ChF+j>#zPRMRhmNNY+3o8Ddr_v2OdyE8$T3xJ`F%+umZ z9-{ZziZd|{4U6O1yFm0IV+O%oQP1q=qYHCB?4ON(*9m{s0Ux%SlW+k(|5!)AYEa7) z*M{-&=lFE@yTs&m{tO$HT*aI7b$gcuKn5Vw-?7trm^Xf6O(VFdXR%+AWr24-BMD(t z_72P*+T5s``C$YLr4hn zz6fTCDGqvNTfs7F5;FFGh6qK1(7BQ|t@c6lq@}+wKFZ}mCNd%wWUtND{}mE#@=0&U z$h!%hkpK_7CW|x{h8OhS01H@qDpF~bn2#Merj89rA7|XmtdbJ;%(Pms02jz>wMBRN zIT$@29Jf_dH;Z7Qeo`ec1Ij_?+`e_Din0!WcNgFVZY=c4A-*rv!VI%VNk zMQYLY)MFrxu#`bH$9xZfb?NQF==k-1eY{i=V*P^ubU``^npbADaNpS2)bqA4uB^;Y z(9>}3`NGixwa1aY+9I9PqFPG~N2hOo;wP26Fnytka>zMR<_a&>|=yEgBqii8j0gsm;-)deC^>6Jgfv;CW)f zj^|uYX_xiMa_lJ%FeDN7t$56RwOk*(WnkE_{yL^K{1_a0?Q(-`rW)>(wvbjXza#Il z+@9u60E1})-v#HU7P$#4Wd0{5eAp90ayYx+ZnBLL(UnDVB$nnD{ORgGs7Wp;8q7ZK zgAI9G)1#yK4#v$bG(9}U{bAM3qGbc2l_BrF{=${mOB9<}cs6a~`yYNTFiw@B1p1$& z3ye8C!V>S`ve;b=ZoUR;Y%W;1d+n_N8*BZU`$N*Oy~T0|ii}PF@N^lR?RviYC@$WY zax=|C3IW~*cRey!t`jYu%FRF=BU7L;`{Ok2HU-jG|G1~A3dh9t(HA5ZvU;UEIF4Rp zM^suc_I)RBZf_BWIebZzumT_3v8DsrkpbrOX(+eBCM%fEASI>p0w}2eZpxHp zoVE*A!3OQ9;rJJE6Ubw^-)SvqoE-S5--UBJwQ{>)N(yAv0K3#y#7T&U%r%zRQPWqn zWjTSXcD#X5B#=Tq+XIfpa!NYNI=ZTn7y+{na#B(fE8}lHM3gHVxy`?t5VXLAj-DT% z?~bv#_7lxo!~+fTAm*66?xF_$!A%IFfq_+x)6;El@2Eww3yAwi3!Ga=S~h4$19&NQ zWG)O)R)uUBP{VDabeGLwhuMXkKjWF#g<=&dbi3{DwPWAyYV3(wDOjall4Bm$1gZ?+ z@4~vxzRQFWhGXa3`=!qYY{9fEQH(a;m}Xuh;ruB10SsDKQ6Xr}0=wPhfNE@+7y`qV96jtu0-3#t?Ynw>pbEwv&;h#N0x7cA81tz<|UJbzh zgWGzr{hxT5`RMRkz=FmzhWZA%iL5I(je?6M?b;0 zK;)`a(NJ_~%D4H;ZReEB`7$$v>50`Vn@py@X*51sc#*ub=I~sQ7S1!e?uKec_lgC5 z{&ryD=W0!#;dPvU-r0E3+DF&%^t6A!$IxUlEs#C_-XOxH@p>Et{)|jU>25G?p|0M% zNUQO>y!YE=% zGEPy&2SbgjEE?_c-osip@^V~xJPA2f_F03<2gV)^Me|Am5b-?rk^#FWC_1_Kt9ds{ zC5jOzSLU?FX*a2H?FmAA7{Hedne9<;bnF#jX(S73Lqc!(wTuZL9WTn2BUMwJ&8)4; za-Yyxfj^FOzo1di{T94HdG~z#ZiDQelaREbP6)TmG-&kUZ~G0US3{)B&qxj zi0o)=M}bAY0p@_!ezxjt-n(%LFZ2pkifPmUYYM%)x0kG{vNTG`rS6RU<%>p7cO!f@ z_kbN(c*o18)X~s+ho0+won*C}oxHB6tI6$M=CgkK^<8m<1wGdaoS?Y$ zy%U8rcgzd0?Vph{Iw71>q4!R%X7i%BT{A89Ah6)Z1x2m23MmDx<;Z1;#zHj|+E*x4 zQrTO#1!)ye&gSaKcP2vv1W(-}*7Syy8-F2SKT&;8z~lWZY;fEZemrXWEM9yx{%4Hn zRaIt7g1l-TKJ4Xd+-@t7XcRUHEs8D1qFx{*CB(4{x`d=;|NSW_FQ<-Kfs8lukVxG8A+xwfD zYJDVY=s~&b`v896Hj1(Z7OEK8^|}}HWw1)!&p(r_?nt~OV9ANXBl>M4-xDJKCwg3n124@-AmlT&XB+>uUD%KBcpPU(O-%=%z6he=;`Y)Z-nnyf^#$BY*g#`n z*a}sX0kNW?u+KN4ZRri?iuBLNb^YwFWEw0N+({3OMGWr7!P7lFJyASJM}xkjjG%a2 zfs%&flK28vS4h8Uoz$xKH4*h{s#fuPy`%*A57DW=(4dWUyuA!$raNf{a*o=G+%o&R zu8pF>EnDd65!%1suE<@*|4m_^S}xmZs|b9r&OA*>%hrmU)} zJ)a}F?wcKabhw|bQku9j@m^L5jw@k|86Xx^so&^$KJK5g2*kJ6)6yjSTB^djxq82j zh1t^(bx5#Y`=v7lG5roGF)PmZ=u-lQQ`&KHE)dJyB?%g;i{vrPu29|=L!{Or{BtIb zdOCVK$}QIO-B-g)l4st9vMwR5$?9L*IcZqrX=88C=dXkCBsah1FCN|*3 z0;&xdbA%?Szlb!&^59TS=CV=-xWcs(bzD;iRWs}}4gvNk)sK8=w{ zB7Wj`=sHRSCQkiWbAR4OJZ=P3*RXwBOf6E7#rQrbbpUr)Vq*>VJ%#l)=oK27<0xX? z^ETVD^!;R}all!&s)#BJ!(H_Pk|&tXp%#cvbLMTXrKB`9$M8I(U_6$^b02Q-miI3dY2_@*>yrRdrLqp9PW<2>~1YZ47<2#^JJJhr+!?TDQ9v!%lR& z+}cfn&9R)_JEkQTce{(iJiF=xfm(OKDmqBBPSrE0g(4WP10)Z^@xq{VJ^FNx$W zj>IeP6_sL?>!+q+;=)cHBmG*#fV2sv#*+2Af^bsMY_v~eA*HXQMv^Rhp5bK=u(^m9 zqE|2N!?6j-@VI<<&|9b1d+6a3IunYCP%<{zbu`zKG6ziP#m*G2vKnk@?0p%ry>D++ zS5+<7YR)Ywtf(pMXldE(58alCB6zDzUP~(8S=i0|2I332vuC3)T)ae24Gb(O;JeEk z5GIce40?zr5ZXyK%N7e6G5b+H+)?dz^@_amiIA|taPjD8&p@4gOJ0o(f=e7A{GRz7 z*v)3`zKgx=zAw+L;!hbHoZn22`(Z&URWyP2K!jRUAF2t9YyCLwf{uAf^W`0{sT~6T`|X2N;6N*HV&b#n{z} zDg~1FtoOMZlOMqLDGJ%+*h^PAs%>oo?gInBXs`gwRBTEh82oqag5<{V9G;ug`+`pT_fx5>%cr#dwl?Lm4wQL5 z)7b-(##NTSi3#VUG4ycRVYT-scM*|8-}c}FwCc*v^J}ItIe}PP7yZ1}R@`_Gmmd%K zTwXi*MYT>Ae2QvnnyUKh%6k3t$jAZ$>?s{TRSeItaEF(c3uG5o=JjAsF=~vK-=^b+ z{kCLmx>r(wxd_%BCSZC{eBLmG^>nz$VdH{p#P$UF4?j^vW#YU2SS*`%&>s;#LYN{O zE9=T;8dT{YeC~@R+?9}~^=X}7SqhivJTqej6!*_%8ZZMbPqWta@>HwCgL$7 z#g;1q4i2vFSy@l7+5VXj!O_noQDU`G-?~{RN;Q-vkdl&;b*>``8@oC0Xtfn72?M+K za7&b=UkvdR@8m#wuA<~nCVd`1+R8?youaOX5 z!KCZce`E{Vc6a-BR%%C&9k}0yMTSKMTn-N(=`P;`7-fAb?_&(WU6l+P{jovp7o!Ua zDl1ESw%~rjE-T+1V$ajCr%ZsuyWgD*xcxYwJX&samNIA7**$#iu=;Fd6rY_vkt6b| z@*OifL?q}_kwQ^HrVrRBK|>hRS|5Jvu=?t@tBcK@mJ+SD_>?IOEW)5Z1Y&a~EN2!w zdX8I{xkC$!Yp{k*-5zMfqws>fFJ{K>&u+KpMCZG^L-Sz%H~~p<9_;THxkDB}&G_9Vgri~eRxzwAVRw8<$8%$3_44v*HlrD+9rF}x@G-psjef~BM4vRe%nG{Z9KrkH2s(eMb}x% z>@9fB_#{?|k=`~weR6mDX^DTcyc_AR48v%Nn=7+tik>wgNp;li``MePG-vHuk$y*~ zExwzn27{9s{`GbJ!kArF*Lwro^0HknBu1|;z7s3AWS)_NV1X(rqwPi#@KfxUf&IY7 ziJww?ARw5a(qbYi_>tTA!R>k-Wtm0Wx*7WBV=SoqQI7R3I}Bu~(Br2ZThrAT*@)R5 z4Qh=KVk?%3R!qAm=)lq|N)N}=Ml@#jfRiIZq>{{k8Kf0ppuHO0D0YkajP8?8xm<&x zVuDoaXF63@6Qldp`n*{=8-dQ+1>AZ|bTxS0x~^UhY857+@}*Ul17^ zl5vv;;m*$alKmA;x!fY+7DRlX0ouAZ=v+0oKF2WN1qZoR-i&$Cf3cq(fm1VN(af=4 zJcVn`FiH3FD3I^lHtd@t?@11pdltNBD>G#3BPnR%Sd8#OxCOGrl9xy6w5$Z zn!K$QU!nde-Ohk-s|VEEbal=)&l#`YRtD2DZ7gG`PdpG+!WK<~RrX&(_-STWpvZ#0 z%S$vX-_oNbBvVrJ*bOK8V~O_acgMp6!twW~3Zp&mtXjvEWAoGB*PT@p&~p|@yqpuL zRiG*!mz~sxjq7L@`m?FBY|}a~zelkPU&^hq>85pJo?A4s)DFuTPs=+}kl)$8>HM48 zQUb#Z^v!*hL%QC9P{X6|Ffn2O2SieV%4Y*d7B@xCZFGZ6`$kJ(c`5Fr;J*UcC+5EqKU@8ep-v1I1;Gpr$+tWRiLD#E z$inskbO{9ZW}omq?qc{^_ zJK=qr!FeNtJ)jI+Nas5)DliJV!Anu)c6&5BI(l4pIcxC&(O?dIQJwbBmL(;&G7XWT zF6SM@*(p=<0qeK^!iWB?3L*&TvR;`I!Tmh;ag!wyMfLd<*A^+pc!yKmGUcK7IoonI z?-Q2_722p!SXCNK@APce3*am#(}h1C@Bt5O?%n!)#ecixh`Zx{&C|42`r2GNy$(^D z=x}CwPEG5Vze;I&LqUP61B8+Y;Sq+4L>(IfS-v;30s#RiC`|}B+iS4rzUdoxZPTjV#V`fYDv4~xS&TsPpWoRmW}H2V8;dK0S-=Vvbv|i}4{kBq2z-^X_&2@3PRoi+i${BZZENV>JsP&R%hOZyJsct0BL>rLwv67) z8)o=(n~L-2xE{(hF|sYkA9jKFKQd*AfIz?n3DYvf6BlQzv6(4+K9iV|VSAGOMLnEe z9!yJ{55G4tdK#_62kLvVSQ!fiiUIB>CSNnK>#1j(Dw=H_jMExohUmFI{co7 zBH)-+^4V3m)tek`vBqR%(W~0j-?A)WJsnvf z_dkfw4sTk0JSE4)7TcHY7)DePV0(yUzOV`@X_*apjWoIre*^1g0zM)dIq_HqEa+Fi z?I=e7zYpotwTR9f@U<9zPzl~hSIO)FyAa@nOxG^#GQzebB-6^bF%W6w#iEq5vNHDE zyRrf*#KJ{5i5pe>Ua4*5FfV}`~SK9i>Cemdo>8yBPFr3W2Q!XTBB$h z3pg0%I=J5xkg)&vPgt+~WivMT{0)xQ_N>4(!QY|v_|J+vxI1?RV8`!@S zWTu2rQ6(_Zxx#O+71$e~Gl}+L|2u~8COUN)g>m2nJOjjEdJc*Rn+B7HfZd~d_>oKd zIZ?p)_M5*bG@P6z2UnN<&{DHHB%=iHD|bFWSfN{JU+)Gi9nDQA9kAywkfVWoI>O@vNUN zZ}GUw!^M6lB+3=>t?{`0+HP0mCM}v4WQFaRY4@}&w-##H9|{V(p0j><=^r~zV(27B zncO$#UcN^ra;?1;7KSTK_!(Vj2$8$pC1NQfQCwR2I>o?cwcJrms-`y#?%VOPxWcyR zK0Y*Om*VFiD+Y!>2g|K7uWPREsG%);$2&{|9%f1-Fb1I?A5gGo^1UNH`bG~ z8Y4!7f;6REca3LSH)^SDLcvh9&gXEmaEYvtg}>ssFOVw?c%J8k_7|yzLHj2pePSSB zg((u~o}fU6@%2E<;lWtC$fK?y9{kZ+wYQgEnLkEj7Bqgc+WMZger{!X%D?owSvw)f zW&M`1wDdhKF}Iw4M@LoHb_@~N`1?|(1HG{HcwvO8^KNuzqOq~l{b2Aiv+<2KR!+zD zk9x7ujKAu|955n9kuuusix2>(MurQ2V(!``Msr+`c<{}EuYmN)!{Dj}( z>E_!oDW}E0z$>~(wML!Q5k^DM=zh9jzSq#SICh0r_3?P?+w*isH(TiX*;9{SAcJf& zBjxMwu*vPH#UxJ(G^9T&wX_FKJJU4#f8<^7h~DE=AEnftwcVj1 zu~3XR!V35&17yPp0#H@cbY*J6 z)va0Di_#wlUimu*)Ia_&?1K}8L(y_f z47#OaOr@v8u6|#?FF$YnDA|!ES7wLbMuyxQvut~6UDs$cCRx=^Jw$PWX9X)@phOp7 zN7$Xhh$1Z-qGF%}2mj2g*5D_13k~M3zu0(Q&qaefcK!2f%{r5sdcx|%>$$~JQ`Xju zQeKab)n8Rp{h?~^P$k~~sv3US$rl)gE9Kc5b7N(b2(d6~X*Sb2{U*0XaqRM+%i)2a zpSCsY5LuUih^o;2&r7}Rr_Vi973+R%?!@IG3rtn85C8ecfq>$vZlzUDcD3uZv$hkZ{Pa7*ADFq;c ze#f>Imlu~*RQNem*NWqQtb14-2!xyx5*;t(-&H3eo=kuhfR?zC&(f*2QGy?fSLBn* zo{SvSnf7ylUmhj3d)byp22*T8;~ zndZdet=%;^nMx9+We|Fq+^+0gV+`$%p|)6UFiHTP~Z16_yJc!H-iHzZy-X+m5d8RG>9~2q*}cxCl_SQwO1D z{uNPC;+&@gC?j&_g|@Ic!Zg{TIYy-fKi?|yhwY&oI;L}>1y^J4SgHQ__EmxMa?4Bt zNplh{lQN5KAfV3Gb^0ScBC_}wpIiZE(Iu)W3k0OU=9=#`8Bhng8ATC(FksgLO3qNM zaT?)Sp1^O`z9iRO6y{loM1e_h9#1#GJrPlUk>2zX+fwWotZCsX^Tprd_KkWS1|tnj z!C*J^vQ8J~brSN@GpA#}xJ~uBaXu`6;hZ`c;7;D0OMc_Wq?m&sR5)$M{trh4sPjGwF7?{3+S3-DeN3TV`AiR4gM^RytEp zu+e1CVBkLak3@1^o_r%OPW8(T3xEGx}uT$lgPNWcjiF5+J^?_vn1 z2?8N0#*;OJ`rjD!3Fpzg3 zpg!sTvm^gA&3z3?d?<|jSmO?$|1$93|2!X`aAE!m%Z!=E{c3r)7s19m2+V3-tXiQ} zioliuzGL2Wlo}qKH#J*rRDzs6BOV;_GM#cMM(BwpPr=<;8rosx`c5LSejnezo{i#n zhzD)>+odcUy_&{PR<=u;s7&tCkH_cTdiea0UF*NMJYF@J@QMiDmcoDRAg^Fi-*q*5?45pbE4*5?jgg-0W1FY_dDoK%ulKqur z8Hw^kc8xU3N;pPBkan{F3}Ah|uhzT?v%uOjY8`c$JhppmVyq`ti{~Cf^^|ZJv`AO$ zRf<)Q(ebcgDd(;rZjfCaFdY)xIW?l&ItBa)I>Ue4c8=q4*~zn1VBd(3ADPT#KsH&<$N}~ZdiZ+?aov#IH_TrG5l`aYS)y;u)WgE zHV3BJBSMCOsK-#%QlsW6;sEd?ov@cWPBZ)uAUXOvuFqCWgF1P)45xp)jQgK7{x$i5 zONE0CCMqtzDL81S1CxXHfBRK=lqg8GXWycu4DSv0Ct_--Tq=^=7hYT|BWu_A+y^4* zB|>X5lUIIabC@^dvxcWnV(l{}H;ToYmeOnl}M;m^bmn%||Qg4`%h6Y}S zxI;E7!LNjo0Vf_ql+x}7m6)~vY^5UUUc G`2BxWR?l$& literal 0 HcmV?d00001 From 7664afd4640c4bd1459697c346b814ce1ca2c823 Mon Sep 17 00:00:00 2001 From: nroope Date: Thu, 30 Jul 2026 13:18:06 +0200 Subject: [PATCH 17/22] Cleanup pqlayers (#60) * pqlayer cleanup * fixed torch quantizer clamp bug, renamed variables in tests --- src/pquant/__init__.py | 2 - src/pquant/core/constants.py | 9 + src/pquant/core/keras/layers.py | 1361 ++++++--------- src/pquant/core/keras/quantizer.py | 111 +- src/pquant/core/torch/fit_compress.py | 6 +- .../core/torch/fixed_point_quantizer.py | 2 +- src/pquant/core/torch/hgq_quantizer.py | 11 +- src/pquant/core/torch/layers.py | 1536 ++++++----------- .../core/torch/pruning_methods/fitcompress.py | 8 +- src/pquant/core/torch/pruning_methods/pdp.py | 10 +- src/pquant/core/torch/quantizer.py | 83 +- src/pquant/core/torch/tracing.py | 3 +- src/pquant/core/torch/utils.py | 3 + src/pquant/data_models/quantization_model.py | 8 +- tests/conftest.py | 5 +- tests/run_tests.sh | 3 +- tests/test_keras_compression_layers.py | 147 ++ tests/test_quantizer_parity.py | 357 ++++ tests/test_torch_checkpoint.py | 171 ++ tests/test_torch_pruning_layers.py | 444 ++--- 20 files changed, 2065 insertions(+), 2215 deletions(-) create mode 100644 tests/test_quantizer_parity.py create mode 100644 tests/test_torch_checkpoint.py diff --git a/src/pquant/__init__.py b/src/pquant/__init__.py index 6c27f9d..ecfd1f8 100644 --- a/src/pquant/__init__.py +++ b/src/pquant/__init__.py @@ -33,7 +33,6 @@ get_ebops, get_layer_keep_ratio, get_model_losses, - load_torch_hgq_model, post_training_prune, ) from .core.torch.tracing import check_quantization, print_quantization_check @@ -64,7 +63,6 @@ _forwards.append("load_from_file") _forwards.append("load_from_dictionary") _forwards.append("get_ebops") - _forwards.append("load_torch_hgq_model") _forwards.append("check_quantization") _forwards.append("print_quantization_check") _forwards.append("PQConfig") diff --git a/src/pquant/core/constants.py b/src/pquant/core/constants.py index 6714d98..af06610 100644 --- a/src/pquant/core/constants.py +++ b/src/pquant/core/constants.py @@ -1,3 +1,5 @@ +from enum import Enum + import optuna from pquant.data_models.pruning_model import ( @@ -11,6 +13,13 @@ WandaPruningModel, ) + +class QuantizationGranularity(str, Enum): + PER_TENSOR = "per_tensor" + PER_CHANNEL = "per_channel" + PER_WEIGHT = "per_weight" + + PRUNING_MODEL_REGISTRY = { "cs": CSPruningModel, "dst": DSTPruningModel, diff --git a/src/pquant/core/keras/layers.py b/src/pquant/core/keras/layers.py index 5615d80..dc996e2 100644 --- a/src/pquant/core/keras/layers.py +++ b/src/pquant/core/keras/layers.py @@ -31,6 +31,30 @@ T = TypeVar("T") +def resolve_data_quant_bits(quant_bits, config): + """Return (k, i, f) from an explicit tuple, or the config's data-lane defaults.""" + if quant_bits is not None: + return quant_bits + parameters = config.quantization_parameters + return ( + parameters.default_data_keep_negatives, + parameters.default_data_integer_bits, + parameters.default_data_fractional_bits, + ) + + +def resolve_weight_quant_bits(quant_bits, config): + """Return (k, i, f) from an explicit tuple, or the config's weight defaults.""" + if quant_bits is not None: + return quant_bits + parameters = config.quantization_parameters + return ( + parameters.default_weight_keep_negatives, + parameters.default_weight_integer_bits, + parameters.default_weight_fractional_bits, + ) + + @keras.saving.register_keras_serializable(package="PQuantML") class PQWeightBiasBase(keras.layers.Layer): def __init__( @@ -54,32 +78,10 @@ def __init__( super().__init__(**kwargs) if isinstance(config, dict): config = PQConfig.load_from_config(config) - if in_quant_bits is not None: - self.k_input, self.i_input, self.f_input = in_quant_bits - else: - self.k_input = config.quantization_parameters.default_data_keep_negatives - self.i_input = config.quantization_parameters.default_data_integer_bits - self.f_input = config.quantization_parameters.default_data_fractional_bits - - if weight_quant_bits is not None: - self.k_weight, self.i_weight, self.f_weight = weight_quant_bits - else: - self.k_weight = config.quantization_parameters.default_weight_keep_negatives - self.i_weight = config.quantization_parameters.default_weight_integer_bits - self.f_weight = config.quantization_parameters.default_weight_fractional_bits - if bias_quant_bits is not None: - self.k_bias, self.i_bias, self.f_bias = bias_quant_bits - else: - self.k_bias = config.quantization_parameters.default_weight_keep_negatives - self.i_bias = config.quantization_parameters.default_weight_integer_bits - self.f_bias = config.quantization_parameters.default_weight_fractional_bits - - if out_quant_bits is not None: - self.k_output, self.i_output, self.f_output = out_quant_bits - else: - self.k_output = config.quantization_parameters.default_data_keep_negatives - self.i_output = config.quantization_parameters.default_data_integer_bits - self.f_output = config.quantization_parameters.default_data_fractional_bits + self.k_input, self.i_input, self.f_input = resolve_data_quant_bits(in_quant_bits, config) + self.k_weight, self.i_weight, self.f_weight = resolve_weight_quant_bits(weight_quant_bits, config) + self.k_bias, self.i_bias, self.f_bias = resolve_weight_quant_bits(bias_quant_bits, config) + self.k_output, self.i_output, self.f_output = resolve_data_quant_bits(out_quant_bits, config) self.layer_type = layer_type self.pruning_layer = get_pruning_layer(config=config, layer_type=self.layer_type) @@ -115,16 +117,15 @@ def __init__( self._is_finetuning = False self.config = config - # Each quantizer follows the config granularity unless its per-quantizer override is set. weight_granularity = weight_quant_granularity if weight_quant_granularity is not None else self.granularity bias_granularity = bias_quant_granularity if bias_quant_granularity is not None else self.granularity in_granularity = in_quant_granularity if in_quant_granularity is not None else self.granularity out_granularity = out_quant_granularity if out_quant_granularity is not None else self.granularity self.weight_quantizer = Quantizer( - k=ops.convert_to_tensor(self.k_weight), - i=ops.convert_to_tensor(self.i_weight), - f=ops.convert_to_tensor(self.f_weight), + k=self.k_weight, + i=self.i_weight, + f=self.f_weight, overflow=self.overflow_mode_parameters, round_mode=self.round_mode, is_heterogeneous=self.use_hgq, @@ -133,12 +134,10 @@ def __init__( hgq_gamma=self.hgq_gamma, place="weight", ) - - # if self.use_bias: self.bias_quantizer = Quantizer( - k=ops.convert_to_tensor(self.k_bias), - i=ops.convert_to_tensor(self.i_bias), - f=ops.convert_to_tensor(self.f_bias), + k=self.k_bias, + i=self.i_bias, + f=self.f_bias, overflow=self.overflow_mode_parameters, round_mode=self.round_mode, is_heterogeneous=self.use_hgq, @@ -148,9 +147,9 @@ def __init__( place="bias", ) self.input_quantizer = Quantizer( - k=ops.convert_to_tensor(self.k_input), - i=ops.convert_to_tensor(self.i_input), - f=ops.convert_to_tensor(self.f_input), + k=self.k_input, + i=self.i_input, + f=self.f_input, overflow=self.overflow_mode_data, round_mode=self.round_mode, is_heterogeneous=self.use_hgq, @@ -161,9 +160,9 @@ def __init__( dynamic_data=self.dynamic_data, ) self.output_quantizer = Quantizer( - k=ops.convert_to_tensor(self.k_output), - i=ops.convert_to_tensor(self.i_output), - f=ops.convert_to_tensor(self.f_output), + k=self.k_output, + i=self.i_output, + f=self.f_output, overflow=self.overflow_mode_data, round_mode=self.round_mode, is_heterogeneous=self.use_hgq, @@ -209,8 +208,60 @@ def build(self, input_shape): ) super().build(input_shape=input_shape) + def _build_quantizers(self, input_shape): + """Build the quantizer lanes in use that are not built yet. The output quantizer is + only created when the output is actually quantized (nothing reads it otherwise).""" + output_shape = self.compute_output_shape(input_shape) + if not self.input_quantizer.built: + self.input_quantizer.build(input_shape) + if not self.weight_quantizer.built: + self.weight_quantizer.build(self._kernel.shape) + if self.use_bias and not self.bias_quantizer.built: + self.bias_quantizer.build(self._bias.shape) + if self.quantize_output and not self.output_quantizer.built: + self.output_quantizer.build(output_shape) + + def _build_pruning_layer(self): + if self.enable_pruning and self.pruning_layer is not None and not self.pruning_layer.built: + pruning_shape = tuple(self._kernel.shape[i] for i in self.weight_transpose) + self.pruning_layer.build(pruning_shape) + + @property + def kernel(self): + if self.final_compression_done: + return self._kernel + if self.pruning_first: + weight = self._prune(self._kernel) + if self.enable_quantization: + weight = self.weight_quantizer(weight) + return weight + weight = self._kernel + if self.enable_quantization: + weight = self.weight_quantizer(weight) + return self._prune(weight) + + @kernel.setter + def kernel(self, kernel): + self._kernel = kernel + + @property + def bias(self): + if self.final_compression_done or self._bias is None: + return self._bias + bias = self._bias + if self.enable_quantization: + bias = self.bias_quantizer(self._bias) + return bias + + @bias.setter + def bias(self, bias): + self._bias = bias + def apply_final_compression(self): - pass + self._kernel.assign(self.kernel) + if self._bias is not None: + self._bias.assign(self.bias) + self.final_compression_done = True def save_own_variables(self, store): if not self.built: @@ -245,16 +296,67 @@ def pre_finetune_function(self): self._is_finetuning = True if hasattr(self, "is_finetuning"): self.is_finetuning.assign(1.0) + if self.pruning_layer is not None: + self.pruning_layer.pre_finetune_function() + + def pre_epoch_function(self, epoch, total_epochs): + if self.enable_pruning: + self.pruning_layer.pre_epoch_function(epoch, total_epochs) - def save_weights(self): + def post_epoch_function(self, epoch, total_epochs, **kwargs): + if self.enable_pruning: + self.pruning_layer.post_epoch_function(epoch, total_epochs, **kwargs) + self._update_pruning_mask() + + def post_round_function(self): + self.pruning_layer.post_round_function() + + def _update_pruning_mask(self): + if self.enable_pruning and hasattr(self.pruning_layer, "update_mask"): + kernel = self._handle_transpose(self._kernel, self.weight_transpose, True) + self.pruning_layer.update_mask(kernel) + + def _save_weights(self): self.init_weight = ops.copy(self._kernel) - def rewind_weights(self): + def _rewind_weights(self): self._kernel.assign(self.init_weight) def ebops(self): return 0.0 + def _masked_weight_bits(self, bw_ker): + """Zero the bit counts of weights that are pruned away or below the quantization step size.""" + mask = self._handle_transpose(self.pruning_layer.get_hard_mask(), self.weight_transpose_back, do_transpose=True) + _, _, f = self.get_weight_quantization_bits() + quantization_step_size = 2 ** (-f - 1) + step_size_mask = ops.cast(ops.abs(self._kernel) > quantization_step_size, self._kernel.dtype) + return bw_ker * mask * step_size_mask + + def _bias_ebops(self): + size = ops.cast(ops.prod(self.input_shape), self.dtype) + bw_bias = self.bias_quantizer.get_total_bits(ops.shape(self._bias)) + return ops.mean(bw_bias) * size + + def _conv_ebops(self, conv_bits, rank, include_mask): + bw_inp = self.input_quantizer.get_total_bits(self.input_shape) + bw_ker = self.weight_quantizer.get_total_bits(ops.shape(self._kernel)) + if include_mask: + bw_ker = self._masked_weight_bits(bw_ker) + if self.parallelization_factor < 0: + ebops = ops.sum(conv_bits(bw_inp, bw_ker)) + else: + if self.do_transpose_data: # channels_last + reduce_axis_input = tuple(range(rank + 1)) + else: + reduce_axis_input = (0,) + tuple(range(2, rank + 2)) + bw_inp = ops.max(bw_inp, axis=reduce_axis_input) + bw_ker = ops.sum(bw_ker, axis=tuple(range(rank))) + ebops = ops.sum(bw_inp[:, None] * bw_ker) + if self.use_bias: + ebops += self._bias_ebops() + return ebops + def hgq_loss(self): if not self.use_hgq: return ops.convert_to_tensor(0.0) @@ -269,39 +371,39 @@ def hgq_loss(self): loss += self.output_quantizer.hgq_loss() return ops.where(ops.cast(self.is_pretraining, "bool"), ops.zeros_like(loss), loss) - def handle_transpose(self, x, transpose, do_transpose=False): + def _handle_transpose(self, x, transpose, do_transpose=False): if do_transpose: x = ops.transpose(x, transpose) return x - def prune(self, weight): + def _prune(self, weight): if self.enable_pruning: - weight = self.handle_transpose(weight, self.weight_transpose, True) + weight = self._handle_transpose(weight, self.weight_transpose, True) weight = self.pruning_layer(weight) - weight = self.handle_transpose(weight, self.weight_transpose_back, True) + weight = self._handle_transpose(weight, self.weight_transpose_back, True) return weight def pre_forward(self, x, training): if self.quantize_input and self.enable_quantization: x = self.input_quantizer(x, training=training) if self.pruning_method == "wanda" and self.enable_pruning: - self.collect_input(x, self._kernel, training) + self._collect_input(x, self._kernel, training) return x - def post_forward(self, x, training): + def _post_forward(self, x, training): if self.quantize_output and self.enable_quantization: x = self.output_quantizer(x, training=training) if self.pruning_method == "activation_pruning" and self.enable_pruning: - self.collect_output(x, training) + self._collect_output(x, training) return x - def collect_input(self, x, weight, training): - collect_x = self.handle_transpose(x, self.data_transpose, self.do_transpose_data) - weight_channels_first = self.handle_transpose(weight, self.weight_transpose, True) + def _collect_input(self, x, weight, training): + collect_x = self._handle_transpose(x, self.data_transpose, self.do_transpose_data) + weight_channels_first = self._handle_transpose(weight, self.weight_transpose, True) self.pruning_layer.collect_input(collect_x, weight_channels_first, training) - def collect_output(self, x, training): - collect_x = self.handle_transpose(x, self.data_transpose, self.do_transpose_data) + def _collect_output(self, x, training): + collect_x = self._handle_transpose(x, self.data_transpose, self.do_transpose_data) self.pruning_layer.collect_output(collect_x, training) @classmethod @@ -413,13 +515,10 @@ def __init__( ) self.depthwise_regularizer = depthwise_regularizer self.use_bias = use_bias - self.strides = strides - self.dilation_rate = dilation_rate self.weight_transpose = (2, 3, 0, 1) self.weight_transpose_back = (2, 3, 0, 1) self.data_transpose = (0, 3, 1, 2) self.do_transpose_data = self.data_format == "channels_last" - self._weight = None self._bias = None def build(self, input_shape): @@ -456,122 +555,30 @@ def build(self, input_shape): ) else: self._bias = None - if self.use_hgq: - self.input_quantizer.build(input_shape) - self.weight_quantizer.build(self._kernel.shape) - if self.use_bias: - self.bias_quantizer.build(self._bias.shape) - self.output_quantizer.build(self.compute_output_shape(input_shape)) - else: - if not self.input_quantizer.built: - self.input_quantizer.build(input_shape) - if not self.weight_quantizer.built: - self.weight_quantizer.build(self._kernel.shape) - if self.use_bias and not self.bias_quantizer.built: - self.bias_quantizer.build(self._bias.shape) - if self.quantize_output and not self.output_quantizer.built: - self.output_quantizer.build(self.compute_output_shape(input_shape)) - self.input_shape = (1,) + input_shape[1:] - if self.enable_pruning and self.pruning_layer is not None and not self.pruning_layer.built: - pruning_shape = tuple(self._kernel.shape[i] for i in self.weight_transpose) - self.pruning_layer.build(pruning_shape) - - @property - def kernel(self): - if self.final_compression_done: - return self._kernel - if self.pruning_first: - weight = self.prune(self._kernel) - if self.enable_quantization: - weight = self.weight_quantizer(weight) - return weight - else: - weight = self._kernel - if self.enable_quantization: - weight = self.weight_quantizer(weight) - return self.prune(weight) - - @kernel.setter - def kernel(self, kernel): - self._kernel = kernel - - @property - def bias(self): - if self.final_compression_done or self._bias is None: - return self._bias - bias = self._bias - if self.enable_quantization: - bias = self.bias_quantizer(self._bias) - return bias - - @bias.setter - def bias(self, bias): - self._bias = bias + self._build_quantizers(input_shape) + self._build_pruning_layer() def ebops(self, include_mask=False): - bw_inp = self.input_quantizer.get_total_bits(self.input_shape) - bw_ker = self.weight_quantizer.get_total_bits(ops.shape(self._kernel)) - if include_mask: - mask = self.handle_transpose(self.pruning_layer.get_hard_mask(), self.weight_transpose_back, do_transpose=True) - bw_ker = bw_ker * mask - _, _, f = self.get_weight_quantization_bits() - quantization_step_size = 2 ** (-f - 1) - step_size_mask = ops.cast((ops.abs(self._kernel) > quantization_step_size), self._kernel.dtype) - bw_ker = bw_ker * step_size_mask - if self.parallelization_factor < 0: - ebops = ops.sum( - ops.depthwise_conv( - bw_inp, - bw_ker, - strides=self.strides, - padding=self.padding, - data_format=None, - dilation_rate=self.dilation_rate, - ) + def conv_bits(bw_inp, bw_ker): + return ops.depthwise_conv( + bw_inp, + bw_ker, + strides=self.strides, + padding=self.padding, + data_format=None, + dilation_rate=self.dilation_rate, ) - else: - reduce_axis_kernel = tuple(range(0, 3)) - if self.data_format == "channels_last": # Is channels last - reduce_axis_input = reduce_axis_kernel - else: - reduce_axis_input = (0,) + tuple(range(2, 4)) - bw_inp = ops.max(bw_inp, axis=reduce_axis_input) - reduce_axis_kernel = tuple(range(0, 2)) - bw_ker = ops.sum(bw_ker, axis=reduce_axis_kernel) - ebops = ops.sum(bw_inp[:, None] * bw_ker) - if self.use_bias: - size = ops.cast(ops.prod(self.input_shape), self.dtype) - bw_bias = self.bias_quantizer.get_total_bits(ops.shape(self._bias)) - ebops += ops.mean(bw_bias) * size - return ebops + + return self._conv_ebops(conv_bits, rank=2, include_mask=include_mask) def call(self, x, training=None): x = self.pre_forward(x, training) x = super().call(x) - x = self.post_forward(x, training) + x = self._post_forward(x, training) if self.use_hgq and self.enable_quantization: self.add_loss(self.hgq_loss()) return x - # Is it supposed to be like this? - def apply_final_compression(self): - self._kernel.assign(self.kernel) - if self._bias is not None: - self._bias.assign(self.bias) - self.final_compression_done = True - - def extra_repr(self) -> str: - """ - Return the extra representation of the module. - """ - return ( - f"in_features={self.in_features} " - f"out_features={self.out_features} " - f"bias={self._bias is not None} " - f"quantize_input={self.quantize_input} " - f"quantize_output={self.quantize_output} " - ) - def _normalize_tuple(value, n): if isinstance(value, int): @@ -672,90 +679,21 @@ def build(self, input_shape): else: self._bias = None super().build(input_shape) - if self.use_hgq: - self.input_quantizer.build(input_shape) - self.weight_quantizer.build(self._kernel.shape) - if self.use_bias: - self.bias_quantizer.build(self._bias.shape) - self.output_quantizer.build(self.compute_output_shape(input_shape)) - else: - if not self.input_quantizer.built: - self.input_quantizer.build(input_shape) - if not self.weight_quantizer.built: - self.weight_quantizer.build(self._kernel.shape) - if self.use_bias and not self.bias_quantizer.built: - self.bias_quantizer.build(self._bias.shape) - if self.quantize_output and not self.output_quantizer.built: - self.output_quantizer.build(self.compute_output_shape(input_shape)) - if self.enable_pruning and self.pruning_layer is not None and not self.pruning_layer.built: - pruning_shape = tuple(self._kernel.shape[i] for i in self.weight_transpose) - self.pruning_layer.build(pruning_shape) - - @property - def kernel(self): - if self.final_compression_done: - return self._kernel - if self.pruning_first: - weight = self.prune(self._kernel) - if self.enable_quantization: - weight = self.weight_quantizer(weight) - return weight - else: - weight = self._kernel - if self.enable_quantization: - weight = self.weight_quantizer(weight) - return self.prune(weight) - - @property - def bias(self): - if self.final_compression_done or self._bias is None: - return self._bias - bias = self._bias - if self.enable_quantization: - bias = self.bias_quantizer(self._bias) - return bias - - @bias.setter - def bias(self, bias): - self._bias = bias + self._build_quantizers(input_shape) + self._build_pruning_layer() def ebops(self, include_mask=False): - bw_inp = self.input_quantizer.get_total_bits(self.input_shape) - bw_ker = self.weight_quantizer.get_total_bits(ops.shape(self._kernel)) - if include_mask: - mask = self.handle_transpose(self.pruning_layer.get_hard_mask(), self.weight_transpose_back, do_transpose=True) - bw_ker = bw_ker * mask - _, _, f = self.get_weight_quantization_bits() - quantization_step_size = 2 ** (-f - 1) - step_size_mask = ops.cast((ops.abs(self._kernel) > quantization_step_size), self._kernel.dtype) - bw_ker = bw_ker * step_size_mask - if self.parallelization_factor < 0: - ebops = ops.sum( - ops.conv( - bw_inp, - bw_ker, - strides=self.strides, - padding=self.padding, - data_format=None, - dilation_rate=self.dilation_rate, - ) + def conv_bits(bw_inp, bw_ker): + return ops.conv( + bw_inp, + bw_ker, + strides=self.strides, + padding=self.padding, + data_format=None, + dilation_rate=self.dilation_rate, ) - else: - reduce_axis_kernel = tuple(range(0, 3)) - if self.do_transpose_data: # Is channels last - reduce_axis_input = reduce_axis_kernel - else: - reduce_axis_input = (0,) + tuple(range(2, 4)) - bw_inp = ops.max(bw_inp, axis=reduce_axis_input) - reduce_axis_kernel = tuple(range(0, 2)) - bw_ker = ops.sum(bw_ker, axis=reduce_axis_kernel) - ebops = ops.sum(bw_inp[:, None] * bw_ker) - if self.use_bias: - size = ops.cast(ops.prod(self.input_shape), self.dtype) - bw_bias = self.bias_quantizer.get_total_bits(ops.shape(self._bias)) - ebops += ops.mean(bw_bias) * size - return ebops + return self._conv_ebops(conv_bits, rank=2, include_mask=include_mask) def compute_output_shape(self, input_shape): return compute_conv_output_shape( @@ -768,12 +706,6 @@ def compute_output_shape(self, input_shape): dilation_rate=self.dilation_rate, ) - def apply_final_compression(self): - self._kernel.assign(self.kernel) - if self._bias is not None: - self._bias.assign(self.bias) - self.final_compression_done = True - def call(self, x, training=None): x = self.pre_forward(x, training) x = ops.conv( @@ -787,7 +719,7 @@ def call(self, x, training=None): if self.use_bias: bias_shape = (1, 1, 1, self.filters) if self.data_format == "channels_last" else (1, self.filters, 1, 1) x = x + ops.reshape(self.bias, bias_shape) - x = self.post_forward(x, training) + x = self._post_forward(x, training) if self.use_hgq and self.enable_quantization: self.add_loss(self.hgq_loss()) return x @@ -882,18 +814,51 @@ def __init__( ) self.do_transpose_data = data_format == "channels_last" - def build(self, input_shape): - super().build(input_shape) - def apply_final_compression(self): self.depthwise_conv.apply_final_compression() self.pointwise_conv.apply_final_compression() + def post_pre_train_function(self): + self.depthwise_conv.post_pre_train_function() + self.pointwise_conv.post_pre_train_function() + + def pre_finetune_function(self): + self.depthwise_conv.pre_finetune_function() + self.pointwise_conv.pre_finetune_function() + + def pre_epoch_function(self, epoch, total_epochs): + self.depthwise_conv.pre_epoch_function(epoch, total_epochs) + self.pointwise_conv.pre_epoch_function(epoch, total_epochs) + + def post_epoch_function(self, epoch, total_epochs, **kwargs): + self.depthwise_conv.post_epoch_function(epoch, total_epochs, **kwargs) + self.pointwise_conv.post_epoch_function(epoch, total_epochs, **kwargs) + + def post_round_function(self): + self.depthwise_conv.post_round_function() + self.pointwise_conv.post_round_function() + + def _save_weights(self): + self.depthwise_conv._save_weights() + self.pointwise_conv._save_weights() + + def _rewind_weights(self): + self.depthwise_conv._rewind_weights() + self.pointwise_conv._rewind_weights() + def call(self, x, training=None): x = self.depthwise_conv(x, training=training) x = self.pointwise_conv(x, training=training) return x + @classmethod + def from_config(cls, config): + final_compression_done = config.pop("final_compression_done", False) + instance = cls(**config) + instance.depthwise_conv.final_compression_done = final_compression_done + instance.pointwise_conv.final_compression_done = final_compression_done + return instance + def get_config(self): config = super().get_config() config.update( @@ -909,6 +874,7 @@ def get_config(self): "use_bias": self.pointwise_conv.use_bias, "quantize_input": self.depthwise_conv.quantize_input, "quantize_output": self.pointwise_conv.quantize_output, + "final_compression_done": self.depthwise_conv.final_compression_done, } ) return config @@ -1007,89 +973,21 @@ def build(self, input_shape): else: self._bias = None super().build(input_shape) - if self.use_hgq: - self.input_quantizer.build(input_shape) - self.weight_quantizer.build(self._kernel.shape) - if self.use_bias: - self.bias_quantizer.build(self._bias.shape) - self.output_quantizer.build(self.compute_output_shape(input_shape)) - else: - if not self.input_quantizer.built: - self.input_quantizer.build(input_shape) - if not self.weight_quantizer.built: - self.weight_quantizer.build(self._kernel.shape) - if self.use_bias and not self.bias_quantizer.built: - self.bias_quantizer.build(self._bias.shape) - if self.quantize_output and not self.output_quantizer.built: - self.output_quantizer.build(self.compute_output_shape(input_shape)) - if self.enable_pruning and self.pruning_layer is not None and not self.pruning_layer.built: - pruning_shape = tuple(self._kernel.shape[i] for i in self.weight_transpose) - self.pruning_layer.build(pruning_shape) - - @property - def kernel(self): - if self.final_compression_done: - return self._kernel - if self.pruning_first: - weight = self.prune(self._kernel) - if self.enable_quantization: - weight = self.weight_quantizer(weight) - return weight - else: - weight = self._kernel - if self.enable_quantization: - weight = self.weight_quantizer(weight) - return self.prune(weight) - - @property - def bias(self): - if self.final_compression_done or self._bias is None: - return self._bias - bias = self._bias - if self.enable_quantization: - bias = self.bias_quantizer(self._bias) - return bias - - @bias.setter - def bias(self, bias): - self._bias = bias + self._build_quantizers(input_shape) + self._build_pruning_layer() def ebops(self, include_mask=False): - bw_inp = self.input_quantizer.get_total_bits(self.input_shape) - bw_ker = self.weight_quantizer.get_total_bits(ops.shape(self._kernel)) - if include_mask: - mask = self.handle_transpose(self.pruning_layer.get_hard_mask(), self.weight_transpose_back, do_transpose=True) - bw_ker = bw_ker * mask - _, _, f = self.get_weight_quantization_bits() - quantization_step_size = 2 ** (-f - 1) - step_size_mask = ops.cast((ops.abs(self._kernel) > quantization_step_size), self._kernel.dtype) - bw_ker = bw_ker * step_size_mask - if self.parallelization_factor < 0: - ebops = ops.sum( - ops.conv( - bw_inp, - bw_ker, - strides=self.strides, - padding=self.padding, - data_format=None, - dilation_rate=self.dilation_rate, - ) + def conv_bits(bw_inp, bw_ker): + return ops.conv( + bw_inp, + bw_ker, + strides=self.strides, + padding=self.padding, + data_format=None, + dilation_rate=self.dilation_rate, ) - else: - reduce_axis_kernel = tuple(range(0, 2)) - if self.do_transpose_data: # Is channels last - reduce_axis_input = reduce_axis_kernel - else: - reduce_axis_input = (0,) + tuple(range(2, 3)) - bw_inp = ops.max(bw_inp, axis=reduce_axis_input) - reduce_axis_kernel = tuple(range(0, 1)) - bw_ker = ops.sum(bw_ker, axis=reduce_axis_kernel) - ebops = ops.sum(bw_inp[:, None] * bw_ker) - if self.use_bias: - size = ops.cast(ops.prod(self.input_shape), self.dtype) - bw_bias = self.bias_quantizer.get_total_bits(ops.shape(self._bias)) - ebops += ops.mean(bw_bias) * size - return ebops + + return self._conv_ebops(conv_bits, rank=1, include_mask=include_mask) def compute_output_shape(self, input_shape): return compute_conv_output_shape( @@ -1102,12 +1000,6 @@ def compute_output_shape(self, input_shape): dilation_rate=self.dilation_rate, ) - def apply_final_compression(self): - self._kernel.assign(self.kernel) - if self._bias is not None: - self._bias.assign(self.bias) - self.final_compression_done = True - def call(self, x, training=None): x = self.pre_forward(x, training) x = ops.conv( @@ -1121,7 +1013,7 @@ def call(self, x, training=None): if self.use_bias: bias_shape = (1, 1, self.filters) if self.data_format == "channels_last" else (1, self.filters, 1) x = x + ops.reshape(self.bias, bias_shape) - x = self.post_forward(x, training) + x = self._post_forward(x, training) if self.use_hgq and self.enable_quantization: self.add_loss(self.hgq_loss()) return x @@ -1204,7 +1096,6 @@ def __init__( self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.input_spec = InputSpec(min_ndim=2) - self._ebops = self.add_variable(shape=(), initializer="zeros", trainable=False) def build(self, input_shape): input_dim = input_shape[-1] @@ -1226,66 +1117,20 @@ def build(self, input_shape): else: self._bias = None super().build(input_shape) - if not self.input_quantizer.built: - self.input_quantizer.build(input_shape) - if not self.weight_quantizer.built: - self.weight_quantizer.build(self._kernel.shape) - if self.use_bias and not self.bias_quantizer.built: - self.bias_quantizer.build(self._bias.shape) - if self.quantize_output and not self.output_quantizer.built: - output_shape = input_shape[:-1] + (self.units,) - self.output_quantizer.build(output_shape) - if self.enable_pruning and self.pruning_layer is not None and not self.pruning_layer.built: - pruning_shape = tuple(self._kernel.shape[i] for i in self.weight_transpose) - self.pruning_layer.build(pruning_shape) - - @property - def kernel(self): - if self.final_compression_done: - return self._kernel - if self.pruning_first: - weight = self.prune(self._kernel) - if self.enable_quantization: - weight = self.weight_quantizer(weight) - return weight - else: - weight = self._kernel - if self.enable_quantization: - weight = self.weight_quantizer(weight) - return self.prune(weight) - - @property - def bias(self): - if self.final_compression_done or self._bias is None: - return self._bias - bias = self._bias - if self.enable_quantization: - bias = self.bias_quantizer(self._bias) - return bias + self._build_quantizers(input_shape) + self._build_pruning_layer() def ebops(self, include_mask=False): bw_inp = self.input_quantizer.get_total_bits(self.input_shape) bw_ker = self.weight_quantizer.get_total_bits(ops.shape(self._kernel)) if include_mask: - mask = self.handle_transpose(self.pruning_layer.get_hard_mask(), self.weight_transpose_back, do_transpose=True) - bw_ker = bw_ker * mask - _, _, f = self.get_weight_quantization_bits() - quantization_step_size = 2 ** (-f - 1) - step_size_mask = ops.cast((ops.abs(self._kernel) > quantization_step_size), self._kernel.dtype) - bw_ker = bw_ker * step_size_mask + bw_ker = self._masked_weight_bits(bw_ker) ebops = ops.sum(ops.matmul(bw_inp, bw_ker)) if self.use_bias: bw_bias = self.bias_quantizer.get_total_bits(ops.shape(self._bias)) size = ops.cast(ops.prod(self.input_shape[:-1]) * self.units, self.dtype) ebops += ops.mean(bw_bias) * size - ebops = ebops * self.parallelization_factor / self.n_parallel - return ebops - - def apply_final_compression(self): - self._kernel.assign(self.kernel) - if self._bias is not None: - self._bias.assign(self.bias) - self.final_compression_done = True + return ebops * self.parallelization_factor / self.n_parallel def compute_output_shape(self, input_shape): output_shape = list(input_shape) @@ -1293,13 +1138,11 @@ def compute_output_shape(self, input_shape): return tuple(output_shape) def call(self, x, training=None): - self.training = training x = self.pre_forward(x, training) x = ops.matmul(x, self.kernel) - bias = self.bias if self.use_bias: - x = ops.add(x, bias) - x = self.post_forward(x, training) + x = ops.add(x, self.bias) + x = self._post_forward(x, training) if self.use_hgq: self.add_loss(self.hgq_loss()) return x @@ -1339,20 +1182,20 @@ def __init__( if isinstance(config, dict): config = PQConfig.load_from_config(config) super().__init__( - axis, - momentum, - epsilon, - center, - scale, - beta_initializer, - gamma_initializer, - moving_mean_initializer, - moving_variance_initializer, - beta_regularizer, - gamma_regularizer, - beta_constraint, - gamma_constraint, - synchronized, + axis=axis, + momentum=momentum, + epsilon=epsilon, + center=center, + scale=scale, + beta_initializer=beta_initializer, + gamma_initializer=gamma_initializer, + moving_mean_initializer=moving_mean_initializer, + moving_variance_initializer=moving_variance_initializer, + beta_regularizer=beta_regularizer, + gamma_regularizer=gamma_regularizer, + beta_constraint=beta_constraint, + gamma_constraint=gamma_constraint, + synchronized=synchronized, **kwargs, ) self.overflow_mode_parameters = config.quantization_parameters.overflow_mode_parameters @@ -1399,32 +1242,32 @@ def build(self, input_shape): round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=True, + granularity=in_granularity, hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.dynamic_data, - granularity=in_granularity, ) self.weight_quantizer = Quantizer( k=1.0, i=self.i_weight, f=self.f_weight, - round_mode=self.round_mode, overflow=self.overflow_mode_parameters, - is_data=False, + round_mode=self.round_mode, is_heterogeneous=self.use_hgq, - place="weight", + is_data=False, granularity=weight_granularity, + place="weight", ) self.bias_quantizer = Quantizer( k=1.0, i=self.i_bias, f=self.f_bias, - round_mode=self.round_mode, overflow=self.overflow_mode_parameters, - is_data=False, + round_mode=self.round_mode, is_heterogeneous=self.use_hgq, - place="bias", + is_data=False, granularity=bias_granularity, + place="bias", ) self.input_quantizer.build(input_shape) self.weight_quantizer.build(self.moving_variance.shape) @@ -1514,7 +1357,8 @@ def call(self, inputs, training=None, mask=None): scale=gamma, epsilon=self.epsilon, ) - self.add_loss(self.hgq_loss()) + if self.use_hgq and self.enable_quantization: + self.add_loss(self.hgq_loss()) return ops.cast(outputs, self.compute_dtype) def get_input_quantization_bits(self): @@ -1576,20 +1420,8 @@ def __init__( self.out_quant_bits = out_quant_bits self.in_quant_granularity = in_quant_granularity self.out_quant_granularity = out_quant_granularity - - if in_quant_bits is not None: - self.k_input, self.i_input, self.f_input = in_quant_bits - else: - self.k_input = config.quantization_parameters.default_data_keep_negatives - self.i_input = config.quantization_parameters.default_data_integer_bits - self.f_input = config.quantization_parameters.default_data_fractional_bits - - if out_quant_bits is not None: - self.k_output, self.i_output, self.f_output = out_quant_bits - else: - self.k_output = config.quantization_parameters.default_data_keep_negatives - self.i_output = config.quantization_parameters.default_data_integer_bits - self.f_output = config.quantization_parameters.default_data_fractional_bits + self.k_input, self.i_input, self.f_input = resolve_data_quant_bits(in_quant_bits, config) + self.k_output, self.i_output, self.f_output = resolve_data_quant_bits(out_quant_bits, config) self.overflow_mode_data = config.quantization_parameters.overflow_mode_data self.config = config self.round_mode = config.quantization_parameters.round_mode @@ -1631,10 +1463,10 @@ def build(self, input_shape): round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=True, + granularity=in_granularity, hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.dynamic_data, - granularity=in_granularity, ) self.output_quantizer = Quantizer( k=1.0, @@ -1644,10 +1476,10 @@ def build(self, input_shape): round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=True, + granularity=out_granularity, hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.dynamic_data, - granularity=out_granularity, ) self.input_quantizer.build(input_shape) self.output_quantizer.build(self.compute_output_shape(input_shape)) @@ -1668,12 +1500,12 @@ def compute_output_shape(self, input_shape): self.data_format, ) - def pre_pooling(self, x, training): + def _pre_pooling(self, x, training): if self.quantize_input and self.enable_quantization: x = self.input_quantizer(x, training=training) return x - def post_pooling(self, x, training): + def _post_pooling(self, x, training): if self.quantize_output and self.enable_quantization: x = self.output_quantizer(x, training=training) return x @@ -1743,9 +1575,9 @@ def __init__( ) def call(self, x, training=None): - x = self.pre_pooling(x, training) + x = self._pre_pooling(x, training) x = super().call(x) - x = self.post_pooling(x, training) + x = self._post_pooling(x, training) if self.use_hgq and self.enable_quantization: self.add_loss(self.hgq_loss()) return x @@ -1788,9 +1620,9 @@ def __init__( ) def call(self, x, training=None): - x = self.pre_pooling(x, training) + x = self._pre_pooling(x, training) x = super().call(x) - x = self.post_pooling(x, training) + x = self._post_pooling(x, training) if self.use_hgq and self.enable_quantization: self.add_loss(self.hgq_loss()) return x @@ -1925,14 +1757,38 @@ def post_pre_train_function(self): proj.post_pre_train_function() self.softmax.post_pre_train_function() + def pre_finetune_function(self): + for proj in (self.q_proj, self.k_proj, self.v_proj, self.out_proj): + proj.pre_finetune_function() + + def pre_epoch_function(self, epoch, total_epochs): + for proj in (self.q_proj, self.k_proj, self.v_proj, self.out_proj): + proj.pre_epoch_function(epoch, total_epochs) + + def post_epoch_function(self, epoch, total_epochs, **kwargs): + for proj in (self.q_proj, self.k_proj, self.v_proj, self.out_proj): + proj.post_epoch_function(epoch, total_epochs, **kwargs) + + def post_round_function(self): + for proj in (self.q_proj, self.k_proj, self.v_proj, self.out_proj): + proj.post_round_function() + + def _save_weights(self): + for proj in (self.q_proj, self.k_proj, self.v_proj, self.out_proj): + proj._save_weights() + + def _rewind_weights(self): + for proj in (self.q_proj, self.k_proj, self.v_proj, self.out_proj): + proj._rewind_weights() + def _head_bits(self, proj, seq_len): """Bitwidths of a projection's output, in per-head layout (1, H, seq, head_dim).""" bw = proj.output_quantizer.get_total_bits((1, seq_len, self.embed_dim)) bw = ops.reshape(bw, (1, seq_len, self.num_heads, self.head_dim)) return ops.transpose(bw, (0, 2, 1, 3)) - def _attention_ebops(self): - """EBOPs of the q @ k^T and attn @ v einsums (mirrors HGQ's QMultiHeadAttention._compute_ebops).""" + def attention_ebops(self): + """EBOPs of the q @ k^T and attn @ v einsums (mirrors HGQ's QMultiHeadAttention).""" attn_shape = self.softmax.input_shape # (1, H, T, S), stored when the softmax was built query_len, key_len = attn_shape[2], attn_shape[3] bw_q = self._head_bits(self.q_proj, query_len) @@ -1945,7 +1801,7 @@ def _attention_ebops(self): return ebops_qk + ebops_av def ebops(self): - ebops = self._attention_ebops() + self.softmax.ebops() + ebops = self.attention_ebops() + self.softmax.ebops() for proj in (self.q_proj, self.k_proj, self.v_proj, self.out_proj): ebops += proj.ebops(include_mask=proj.enable_pruning) return ebops @@ -1953,7 +1809,7 @@ def ebops(self): def hgq_loss(self): if self.is_pretraining or not self.use_hgq: return ops.convert_to_tensor(0.0) - return ops.convert_to_tensor(self.hgq_beta * self._attention_ebops() + self.softmax.hgq_loss()) + return ops.convert_to_tensor(self.hgq_beta * self.attention_ebops() + self.softmax.hgq_loss()) def call( self, @@ -2020,7 +1876,7 @@ def call( out = ops.reshape(out, (batch_size, query_len, self.embed_dim)) out = self.out_proj(out, training=training) - if self.use_hgq: + if self.use_hgq and self.enable_quantization: self.add_loss(self.hgq_loss()) if need_weights: @@ -2065,19 +1921,36 @@ def from_config(cls, config): return cls(**config) +LAYERS_WITH_PRUNING_LAYER = (PQWeightBiasBase, PQSeparableConv2d, PQMultiheadAttention) + + +def _iter_weight_layers(model): + for layer in model.layers: + if isinstance(layer, PQWeightBiasBase): + yield layer + elif isinstance(layer, PQSeparableConv2d): + yield layer.depthwise_conv + yield layer.pointwise_conv + elif isinstance(layer, PQMultiheadAttention): + yield layer.q_proj + yield layer.k_proj + yield layer.v_proj + yield layer.out_proj + + def call_post_round_functions(model, rewind, rounds, r): last_round = r == rounds - 1 if rewind == "every-round": - rewind_weights_functions(model) + _rewind_weights_functions(model) elif rewind == "post-training-stage" and last_round: - rewind_weights_functions(model) + _rewind_weights_functions(model) elif not last_round: - post_round_functions(model) + _post_round_functions(model) def apply_final_compression(model): for layer in model.layers: - if isinstance(layer, (PQWeightBiasBase, PQSeparableConv2d, PQBatchNormalization, PQDepthwiseConv2d)): + if isinstance(layer, (PQWeightBiasBase, PQSeparableConv2d, PQBatchNormalization)): layer.apply_final_compression() if hasattr(layer, "input_quantizer"): layer.input_quantizer.apply_final_compression() @@ -2089,122 +1962,40 @@ def apply_final_compression(model): return model -def _update_pruning_mask(layer): - if layer.enable_pruning and hasattr(layer.pruning_layer, "update_mask"): - kernel = layer.handle_transpose(layer._kernel, layer.weight_transpose, True) - layer.pruning_layer.update_mask(kernel) - - def post_epoch_functions(model, epoch, total_epochs, **kwargs): for layer in model.layers: - if isinstance( - layer, - ( - PQDepthwiseConv2d, - PQConv2d, - PQConv1d, - PQDense, - ), - ): - if layer.enable_pruning: - layer.pruning_layer.post_epoch_function(epoch, total_epochs, **kwargs) - _update_pruning_mask(layer) - elif isinstance(layer, PQSeparableConv2d): - if layer.enable_pruning: - layer.depthwise_conv.pruning_layer.post_epoch_function(epoch, total_epochs, **kwargs) - _update_pruning_mask(layer.depthwise_conv) - layer.pointwise_conv.pruning_layer.post_epoch_function(epoch, total_epochs, **kwargs) - _update_pruning_mask(layer.pointwise_conv) + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): + layer.post_epoch_function(epoch, total_epochs, **kwargs) def pre_epoch_functions(model, epoch, total_epochs): for layer in model.layers: - if isinstance( - layer, - ( - PQDepthwiseConv2d, - PQConv2d, - PQConv1d, - PQDense, - ), - ): - if layer.enable_pruning: - layer.pruning_layer.pre_epoch_function(epoch, total_epochs) - elif isinstance(layer, PQSeparableConv2d): - if layer.enable_pruning: - layer.depthwise_conv.pruning_layer.pre_epoch_function(epoch, total_epochs) - layer.pointwise_conv.pruning_layer.pre_epoch_function(epoch, total_epochs) + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): + layer.pre_epoch_function(epoch, total_epochs) -def post_round_functions(model): +def _post_round_functions(model): for layer in model.layers: - if isinstance( - layer, - ( - PQDepthwiseConv2d, - PQConv2d, - PQConv1d, - PQDense, - ), - ): - layer.pruning_layer.post_round_function() - elif isinstance(layer, PQSeparableConv2d): - layer.depthwise_conv.pruning_layer.post_round_function() - layer.pointwise_conv.pruning_layer.post_round_function() + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): + layer.post_round_function() def save_weights_functions(model): for layer in model.layers: - if isinstance( - layer, - ( - PQDepthwiseConv2d, - PQConv2d, - PQConv1d, - PQDense, - ), - ): - layer.save_weights() - elif isinstance(layer, PQSeparableConv2d): - layer.depthwise_conv.save_weights() - layer.pointwise_conv.save_weights() + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): + layer._save_weights() -def rewind_weights_functions(model): +def _rewind_weights_functions(model): for layer in model.layers: - if isinstance( - layer, - ( - PQDepthwiseConv2d, - PQConv2d, - PQConv1d, - PQDense, - ), - ): - layer.rewind_weights() - elif isinstance(layer, PQSeparableConv2d): - layer.depthwise_conv.rewind_weights() - layer.pointwise_conv.rewind_weights() + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): + layer._rewind_weights() def pre_finetune_functions(model): for layer in model.layers: - if isinstance( - layer, - ( - PQDepthwiseConv2d, - PQConv2d, - PQConv1d, - PQDense, - ), - ): + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): layer.pre_finetune_function() - layer.pruning_layer.pre_finetune_function() - elif isinstance(layer, PQSeparableConv2d): - layer.depthwise_conv.pre_finetune_function() - layer.depthwise_conv.pruning_layer.pre_finetune_function() - layer.pointwise_conv.pre_finetune_function() - layer.pointwise_conv.pruning_layer.pre_finetune_function() def post_pretrain_functions(model, config): @@ -2212,187 +2003,91 @@ def post_pretrain_functions(model, config): if isinstance( layer, ( - PQDepthwiseConv2d, - PQConv2d, - PQConv1d, - PQDense, + PQWeightBiasBase, + PQSeparableConv2d, + PQActivation, + PQAvgPoolBase, + PQBatchNormalization, + PQSoftmax, + PQMultiheadAttention, ), ): layer.post_pre_train_function() - elif isinstance(layer, PQSeparableConv2d): - layer.depthwise_conv.post_pre_train_function() - layer.pointwise_conv.post_pre_train_function() - elif isinstance(layer, (PQActivation, PQAvgPoolBase, PQBatchNormalization, PQSoftmax, PQMultiheadAttention)): - layer.post_pre_train_function() if config.pruning_parameters.pruning_method == "pdp" or ( config.pruning_parameters.pruning_method == "wanda" and config.pruning_parameters.calculate_pruning_budget ): - pdp_setup(model, config) + _pdp_setup(model, config) -def pdp_setup(model, config): +def _pdp_setup(model, config): """ Calculates a global sparsity threshold. Initializes target sparsity for each layer, which depends on how large percentage of weights in the layer is smaller than the global threshold """ - global_weights = None - for layer in model.layers: - if isinstance( - layer, - ( - PQDepthwiseConv2d, - PQConv2d, - PQConv1d, - PQDense, - ), - ): - if global_weights is None: - global_weights = ops.ravel(layer.kernel) - else: - global_weights = ops.concatenate((global_weights, ops.ravel(layer.kernel))) - elif isinstance(layer, PQSeparableConv2d): - if global_weights is None: - global_weights = ops.ravel(layer.depthwise_conv.kernel) - global_weights = ops.concatenate((global_weights, ops.ravel(layer.pointwise_conv.kernel))) - else: - global_weights = ops.concatenate((global_weights, ops.ravel(layer.depthwise_conv.kernel))) - global_weights = ops.concatenate((global_weights, ops.ravel(layer.pointwise_conv.kernel))) - + global_weights = ops.concatenate([ops.ravel(layer.kernel) for layer in _iter_weight_layers(model)]) abs_global_weights = ops.abs(global_weights) global_weight_topk, _ = ops.top_k(abs_global_weights, ops.size(abs_global_weights)) threshold = global_weight_topk[int((1 - config.pruning_parameters.sparsity) * float(ops.size(global_weight_topk)))] global_weights_below_threshold = ops.where(abs_global_weights < threshold, 1, 0) idx = 0 - for layer in model.layers: - if isinstance( - layer, - ( - PQDepthwiseConv2d, - PQConv2d, - PQConv1d, - PQDense, - ), - ): - weight_size = ops.size(layer.kernel) - w = ops.sum(global_weights_below_threshold[idx : idx + weight_size]) - layer.pruning_layer.init_r = ops.convert_to_tensor(w / weight_size, dtype=layer.kernel.dtype) - layer.pruning_layer.sparsity = ops.convert_to_tensor(w / weight_size, dtype=layer.kernel.dtype) # Wanda - idx += weight_size - elif isinstance(layer, PQSeparableConv2d): - weight_size = ops.size(layer.depthwise_conv.kernel) - w = ops.sum(global_weights_below_threshold[idx : idx + weight_size]) - layer.depthwise_conv.pruning_layer.init_r = ops.convert_to_tensor( - w / weight_size, dtype=layer.depthwise_conv.kernel.dtype - ) - layer.depthwise_conv.pruning_layer.sparsity = ops.convert_to_tensor( - w / weight_size, dtype=layer.depthwise_conv.kernel.dtype - ) # Wanda - idx += weight_size - - weight_size = ops.size(layer.pointwise_conv.kernel) - w = ops.sum(global_weights_below_threshold[idx : idx + weight_size]) - layer.pointwise_conv.pruning_layer.init_r = ops.convert_to_tensor( - w / weight_size, dtype=layer.pointwise_conv.kernel.dtype - ) - layer.pointwise_conv.pruning_layer.sparsity = ops.convert_to_tensor( - w / weight_size, dtype=layer.pointwise_conv.kernel.dtype - ) # Wanda - idx += weight_size + for layer in _iter_weight_layers(model): + weight_size = ops.size(layer.kernel) + w = ops.sum(global_weights_below_threshold[idx : idx + weight_size]) + sparsity = ops.convert_to_tensor(w / weight_size, dtype=layer.kernel.dtype) + layer.pruning_layer.init_r = sparsity + layer.pruning_layer.sparsity = sparsity # Wanda + idx += weight_size def get_layer_keep_ratio(model): total_w = 0 remaining_weights = 0 for layer in model.layers: - if isinstance( - layer, - ( - PQDepthwiseConv2d, - PQConv2d, - PQConv1d, - PQDense, - ), - ): + if isinstance(layer, PQWeightBiasBase): weight = layer.kernel total_w += ops.size(weight) - rem = ops.count_nonzero(weight) - remaining_weights += rem - elif isinstance(layer, PQSeparableConv2d): - depthwise_weight = ops.cast(layer.depthwise_conv.kernel, layer.depthwise_conv.kernel.dtype) - pointwise_weight = ops.cast(layer.pointwise_conv.kernel, layer.pointwise_conv.kernel.dtype) - - depthwise_weight = layer.depthwise_conv.kernel - transpose = layer.depthwise_conv.weight_transpose - if layer.depthwise_conv.enable_pruning: - depthwise_weight = layer.depthwise_conv.pruning_layer.get_hard_mask( - ops.transpose(depthwise_weight, transpose) - ) * ops.transpose(depthwise_weight, transpose) - total_w += ops.size(layer.depthwise_conv.kernel) - rem = ops.count_nonzero(depthwise_weight) - remaining_weights += rem - - pointwise_weight = layer.pointwise_conv.kernel - transpose = layer.pointwise_conv.weight_transpose - if layer.pointwise_conv.enable_pruning: - pointwise_weight = layer.pointwise_conv.pruning_layer.get_hard_mask( - ops.transpose(pointwise_weight, transpose) - ) * ops.transpose(pointwise_weight, transpose) - total_w += ops.size(layer.pointwise_conv.kernel) - rem = ops.count_nonzero(pointwise_weight) - remaining_weights += rem - + remaining_weights += ops.count_nonzero(weight) + elif isinstance(layer, (PQSeparableConv2d, PQMultiheadAttention)): + if isinstance(layer, PQSeparableConv2d): + sublayers = (layer.depthwise_conv, layer.pointwise_conv) + else: + sublayers = (layer.q_proj, layer.k_proj, layer.v_proj, layer.out_proj) + for sublayer in sublayers: + weight = sublayer.kernel + total_w += ops.size(weight) + remaining_weights += ops.count_nonzero(weight) elif isinstance(layer, (Conv2D, Conv1D, DepthwiseConv2D, Dense)): weight = layer.kernel total_w += ops.size(weight) remaining_weights += ops.count_nonzero(weight) elif isinstance(layer, SeparableConv2D): - depthwise_weight = layer.depthwise_kernel - pointwise_weight = layer.pointwise_kernel - total_w += ops.size(depthwise_weight) - total_w += ops.size(pointwise_weight) - remaining_weights += ops.count_nonzero(depthwise_weight) - remaining_weights += ops.count_nonzero(pointwise_weight) + total_w += ops.size(layer.depthwise_kernel) + total_w += ops.size(layer.pointwise_kernel) + remaining_weights += ops.count_nonzero(layer.depthwise_kernel) + remaining_weights += ops.count_nonzero(layer.pointwise_kernel) if total_w != 0: return remaining_weights / total_w return 0.0 -def is_training_stage(layer): - return False if layer.pruning_layer.is_finetuning or layer.pruning_layer.is_pretraining else True +def _is_training_stage(layer): + return not (layer.pruning_layer._is_finetuning or layer.pruning_layer._is_pretraining) def get_model_losses(model, losses): + for layer in _iter_weight_layers(model): + if layer.enable_pruning and _is_training_stage(layer): + losses += layer.pruning_layer.calculate_additional_loss() + if layer.enable_quantization and layer.use_hgq: + losses += layer.hgq_loss() for layer in model.layers: - loss = 0.0 - if isinstance( - layer, - ( - PQDepthwiseConv2d, - PQConv2d, - PQConv1d, - PQDense, - ), - ): - if layer.enable_pruning and is_training_stage(layer): - loss += layer.pruning_layer.calculate_additional_loss() - if layer.enable_quantization and layer.use_hgq: - loss += layer.hgq_loss() - losses += loss - elif isinstance(layer, PQSeparableConv2d): - if layer.enable_pruning and is_training_stage(layer): - loss += layer.depthwise_conv.pruning_layer.calculate_additional_loss() - loss += layer.pointwise_conv.pruning_layer.calculate_additional_loss() - if layer.enable_quantization and layer.use_hgq: - loss += layer.depthwise_conv.hgq_loss() - loss += layer.pointwise_conv.hgq_loss() - losses += loss - elif isinstance(layer, (PQActivation, PQAvgPoolBase, PQBatchNormalization, PQSoftmax)): + if isinstance(layer, (PQActivation, PQAvgPoolBase, PQBatchNormalization, PQSoftmax)): if layer.enable_quantization and layer.use_hgq: losses += layer.hgq_loss() return losses -def check_activation(layer, config): +def _check_activation(layer, config): """ Replaces activations with quantized activations. The activation can be a part of another layer such as Conv2D, or an Activation layer @@ -2409,7 +2104,7 @@ def check_activation(layer, config): else ReLU() ) if quantization_enabled: - set_quantization_bits_activations(config, layer, act) + _set_quantization_bits_activations(config, layer, act) act.build(layer.input.shape) elif layer.activation.__name__ == "tanh": type_of_tanh = "tanh" if config.quantization_parameters.use_real_tanh else "hard_tanh" @@ -2419,13 +2114,22 @@ def check_activation(layer, config): else Activation(activation="tanh") ) if quantization_enabled: - set_quantization_bits_activations(config, layer, act) + _set_quantization_bits_activations(config, layer, act) act.build(layer.input.shape) - else: - act = None return act +def _build_pruning_layer_from_kernel(new_layer, kernel): + transposed_kernel = ops.transpose(kernel, new_layer.weight_transpose) + new_layer.pruning_layer.build(transposed_kernel.shape) + + +def _copy_kernel_and_bias(new_layer, layer): + new_layer._kernel.assign(layer._kernel) + if layer.use_bias: + new_layer._bias.assign(layer.bias) + + def add_compression_layers(model, config, input_shape=None): # Pruning algorithms assume channels_first format # Creates a new functional model from model, replacing certain layers with compressed / quantized variants @@ -2455,17 +2159,11 @@ def add_compression_layers(model, config, input_shape=None): quantize_input=quantize_input, quantize_output=quantize_output, ) - set_quantization_bits_weight_layers(config, layer, new_layer) - - enable_pruning = get_enable_pruning(layer, config) - new_layer.set_enable_pruning(enable_pruning) - pruning_layer_input = layer.kernel - transpose_shape = new_layer.weight_transpose - pruning_layer_input = ops.transpose(pruning_layer_input, transpose_shape) - new_layer.pruning_layer.build(pruning_layer_input.shape) - + _set_quantization_bits_weight_layers(config, layer, new_layer) + new_layer.set_enable_pruning(_get_enable_pruning(layer, config)) + _build_pruning_layer_from_kernel(new_layer, layer.kernel) x = new_layer(x) - act = check_activation(layer, config) + act = _check_activation(layer, config) elif isinstance(layer, Conv2D): new_layer = PQConv2d( config=config, @@ -2487,19 +2185,13 @@ def add_compression_layers(model, config, input_shape=None): quantize_input=quantize_input, quantize_output=quantize_output, ) - set_quantization_bits_weight_layers(config, layer, new_layer) - enable_pruning = get_enable_pruning(layer, config) - new_layer.set_enable_pruning(enable_pruning) - pruning_layer_input = layer.kernel - transpose_shape = new_layer.weight_transpose - pruning_layer_input = ops.transpose(pruning_layer_input, transpose_shape) - new_layer.pruning_layer.build(pruning_layer_input.shape) + _set_quantization_bits_weight_layers(config, layer, new_layer) + new_layer.set_enable_pruning(_get_enable_pruning(layer, config)) + _build_pruning_layer_from_kernel(new_layer, layer.kernel) new_layer.build(x.shape) x = new_layer(x) - new_layer._kernel.assign(layer._kernel) - if layer.use_bias: - new_layer._bias.assign(layer.bias) - act = check_activation(layer, config) + _copy_kernel_and_bias(new_layer, layer) + act = _check_activation(layer, config) elif isinstance(layer, SeparableConv2D): new_layer = PQSeparableConv2d( config, @@ -2523,26 +2215,18 @@ def add_compression_layers(model, config, input_shape=None): quantize_input=quantize_input, quantize_output=quantize_output, ) - set_quantization_bits_weight_layers(config, layer, new_layer) + _set_quantization_bits_weight_layers(config, layer, new_layer) - enable_pruning_depthwise, enable_pruning_pointwise = get_enable_pruning(layer, config) + enable_pruning_depthwise, enable_pruning_pointwise = _get_enable_pruning(layer, config) new_layer.depthwise_conv.set_enable_pruning(enable_pruning_depthwise) new_layer.pointwise_conv.set_enable_pruning(enable_pruning_pointwise) - - pruning_layer_input = layer.depthwise_kernel - pruning_layer_input = ops.transpose(pruning_layer_input, new_layer.depthwise_conv.weight_transpose) - new_layer.depthwise_conv.pruning_layer.build(pruning_layer_input.shape) - - pointwise_pruning_layer_input = layer.pointwise_kernel - pointwise_pruning_layer_input = ops.transpose( - pointwise_pruning_layer_input, new_layer.pointwise_conv.weight_transpose - ) - new_layer.pointwise_conv.pruning_layer.build(pointwise_pruning_layer_input.shape) + _build_pruning_layer_from_kernel(new_layer.depthwise_conv, layer.depthwise_kernel) + _build_pruning_layer_from_kernel(new_layer.pointwise_conv, layer.pointwise_kernel) new_layer.depthwise_conv.build(x.shape) y = new_layer.depthwise_conv(x).shape new_layer.pointwise_conv.build(y) x = new_layer(x) - act = check_activation(layer, config) + act = _check_activation(layer, config) elif isinstance(layer, Conv1D): new_layer = PQConv1d( config=config, @@ -2558,19 +2242,13 @@ def add_compression_layers(model, config, input_shape=None): quantize_input=quantize_input, quantize_output=quantize_output, ) - set_quantization_bits_weight_layers(config, layer, new_layer) - enable_pruning = get_enable_pruning(layer, config) - new_layer.set_enable_pruning(enable_pruning) - pruning_layer_input = layer.kernel - transpose_shape = new_layer.weight_transpose - pruning_layer_input = ops.transpose(pruning_layer_input, transpose_shape) - new_layer.pruning_layer.build(pruning_layer_input.shape) + _set_quantization_bits_weight_layers(config, layer, new_layer) + new_layer.set_enable_pruning(_get_enable_pruning(layer, config)) + _build_pruning_layer_from_kernel(new_layer, layer.kernel) new_layer.build(x.shape) x = new_layer(x) - new_layer._kernel.assign(layer._kernel) - if layer.use_bias: - new_layer._bias.assign(layer.bias) - act = check_activation(layer, config) + _copy_kernel_and_bias(new_layer, layer) + act = _check_activation(layer, config) elif isinstance(layer, Dense): new_layer = PQDense( config=config, @@ -2586,30 +2264,24 @@ def add_compression_layers(model, config, input_shape=None): quantize_input=quantize_input, quantize_output=quantize_output, ) - set_quantization_bits_weight_layers(config, layer, new_layer) - enable_pruning = get_enable_pruning(layer, config) - new_layer.set_enable_pruning(enable_pruning) - pruning_layer_input = layer.kernel - transpose_shape = new_layer.weight_transpose - pruning_layer_input = ops.transpose(pruning_layer_input, transpose_shape) - new_layer.pruning_layer.build(pruning_layer_input.shape) + _set_quantization_bits_weight_layers(config, layer, new_layer) + new_layer.set_enable_pruning(_get_enable_pruning(layer, config)) + _build_pruning_layer_from_kernel(new_layer, layer.kernel) x = new_layer(x) - new_layer._kernel.assign(layer._kernel) - if layer.use_bias: - new_layer._bias.assign(layer.bias) - act = check_activation(layer, config) + _copy_kernel_and_bias(new_layer, layer) + act = _check_activation(layer, config) # Activation layers elif isinstance(layer, ReLU): if config.quantization_parameters.enable_quantization: new_layer = PQActivation(config, "relu", quantize_input=quantize_input, quantize_output=quantize_output) - set_quantization_bits_activations(config, layer, new_layer) + _set_quantization_bits_activations(config, layer, new_layer) new_layer.build(layer.input.shape) x = new_layer(x) else: x = layer(x) elif isinstance(layer, Activation): - new_layer = check_activation(layer, config) + new_layer = _check_activation(layer, config) if new_layer is not None: x = new_layer(x) @@ -2622,7 +2294,7 @@ def add_compression_layers(model, config, input_shape=None): padding=layer.padding, data_format=layer.data_format, ) - set_quantization_bits_activations(config, layer, new_layer) + _set_quantization_bits_activations(config, layer, new_layer) new_layer.build(x.shape) x = new_layer(x) elif isinstance(layer, AveragePooling2D): @@ -2634,7 +2306,7 @@ def add_compression_layers(model, config, input_shape=None): padding=layer.padding, data_format=layer.data_format, ) - set_quantization_bits_activations(config, layer, new_layer) + _set_quantization_bits_activations(config, layer, new_layer) new_layer.build(x.shape) x = new_layer(x) elif isinstance(layer, (BatchNormalization)): @@ -2657,7 +2329,7 @@ def add_compression_layers(model, config, input_shape=None): layer.synchronized, quantize_input=True, ) - set_quantization_bits_activations(config, layer, new_layer) + _set_quantization_bits_activations(config, layer, new_layer) new_layer.build(x.shape) x = new_layer(x) else: @@ -2670,55 +2342,39 @@ def add_compression_layers(model, config, input_shape=None): return replaced_model -def set_quantization_bits_activations(config, layer, new_layer): - i_input = i_output = i_weight = i_bias = config.quantization_parameters.default_data_integer_bits - f_input = f_output = f_weight = f_bias = config.quantization_parameters.default_data_fractional_bits +def _get_quant_section(section, i_default, f_default, target=None, quantize_attr=None): + """Read integer/fractional bits from one layer_specific section, optionally applying its quantize flag.""" + if section is None: + return i_default, f_default + if quantize_attr is not None and "quantize" in section: + setattr(target, quantize_attr, section["quantize"]) + return section.get("integer_bits", i_default), section.get("fractional_bits", f_default) + + +def _set_quantization_bits_activations(config, layer, new_layer): + quant_params = config.quantization_parameters + i_input = i_output = i_weight = i_bias = quant_params.default_data_integer_bits + f_input = f_output = f_weight = f_bias = quant_params.default_data_fractional_bits if isinstance(layer, ReLU): f_input += 1 f_output += 1 # Unsigned, add 1 bit to default value only - layer_specific = config.quantization_parameters.layer_specific - if layer.name in layer_specific: - layer_config = layer_specific[layer.name] + layer_config = quant_params.layer_specific.get(layer.name) + if layer_config is not None: if hasattr(layer, "activation") and layer.activation.__name__ in layer_config: - if "input" in layer_config[layer.activation.__name__]: - if "integer_bits" in layer_config[layer.activation.__name__]["input"]: - i_input = layer_config[layer.activation.__name__]["input"]["integer_bits"] - if "integer_bits" in layer_config[layer.activation.__name__]["input"]: - f_input = layer_config[layer.activation.__name__]["input"]["fractional_bits"] - if "quantize" in layer_config[layer.activation.__name__]["input"]: - new_layer.quantize_input = layer_config[layer.activation.__name__]["input"]["quantize"] - if "output" in layer_config[layer.activation.__name__]: - if "integer_bits" in layer_config[layer.activation.__name__]["output"]: - i_output = layer_config[layer.activation.__name__]["output"]["integer_bits"] - if "fractional_bits" in layer_config[layer.activation.__name__]["output"]: - f_output = layer_config[layer.activation.__name__]["output"]["fractional_bits"] - if "quantize" in layer_config[layer.activation.__name__]["output"]: - new_layer.quantize_output = layer_config[layer.activation.__name__]["output"]["quantize"] + activation_config = layer_config[layer.activation.__name__] + i_input, f_input = _get_quant_section( + activation_config.get("input"), i_input, f_input, new_layer, "quantize_input" + ) + i_output, f_output = _get_quant_section( + activation_config.get("output"), i_output, f_output, new_layer, "quantize_output" + ) else: - if "input" in layer_config: - if "integer_bits" in layer_config["input"]: - i_input = layer_config["input"]["integer_bits"] - if "fractional_bits" in layer_config["input"]: - f_input = layer_config["input"]["fractional_bits"] - if "quantize" in layer_config["input"]: - new_layer.quantize_input = layer_config["input"]["quantize"] - if "weight" in layer_config: - if "integer_bits" in layer_config["weight"]: - i_weight = layer_config["weight"]["integer_bits"] - if "fractional_bits" in layer_config["weight"]: - f_weight = layer_config["weight"]["fractional_bits"] - if "bias" in layer_config: - if "integer_bits" in layer_config["bias"]: - i_bias = layer_config["bias"]["integer_bits"] - if "fractional_bits" in layer_config["bias"]: - f_bias = layer_config["bias"]["fractional_bits"] - if "output" in layer_config: - if "integer_bits" in layer_config["output"]: - i_output = layer_config["output"]["integer_bits"] - if "fractional_bits" in layer_config["output"]: - f_output = layer_config["output"]["fractional_bits"] - if "quantize" in layer_config["output"]: - new_layer.quantize_output = layer_config["output"]["quantize"] + i_input, f_input = _get_quant_section(layer_config.get("input"), i_input, f_input, new_layer, "quantize_input") + i_weight, f_weight = _get_quant_section(layer_config.get("weight"), i_weight, f_weight) + i_bias, f_bias = _get_quant_section(layer_config.get("bias"), i_bias, f_bias) + i_output, f_output = _get_quant_section( + layer_config.get("output"), i_output, f_output, new_layer, "quantize_output" + ) if isinstance(layer, BatchNormalization): new_layer.i_weight = i_weight new_layer.f_weight = f_weight @@ -2730,86 +2386,59 @@ def set_quantization_bits_activations(config, layer, new_layer): new_layer.f_output = f_output -def set_quantization_bits_weight_layers(config, layer, new_layer): - layer_specific = config.quantization_parameters.layer_specific +def _set_quantization_bits_weight_layers(config, layer, new_layer): + quant_params = config.quantization_parameters + layer_config = quant_params.layer_specific.get(layer.name) if isinstance(layer, SeparableConv2D): - dw_i_bits_w = pw_i_bits_w = pw_i_bits_b = config.quantization_parameters.default_weight_integer_bits - dw_f_bits_w = pw_f_bits_w = pw_f_bits_b = config.quantization_parameters.default_weight_fractional_bits - i_input = i_output = config.quantization_parameters.default_data_integer_bits - f_input = f_output = config.quantization_parameters.default_data_fractional_bits - if layer.name in layer_specific: - layer_config = layer_specific[layer.name] - if "input" in layer_config: - if "quantize" in layer_config["input"]: - new_layer.depthwise_conv.quantize_input = layer_config["input"]["quantize"] - if "integer_bits" in layer_config["input"]: - i_input = layer_config["input"]["integer_bits"] - if "fractional_bits" in layer_config["input"]: - f_input = layer_config["input"]["fractional_bits"] - if "depthwise" in layer_config: - if "weight" in layer_config["depthwise"]: - dw_i_bits_w = layer_config["depthwise"]["weight"]["integer_bits"] - dw_f_bits_w = layer_config["depthwise"]["weight"]["fractional_bits"] - if "pointwise" in layer_config: - if "weight" in layer_config["pointwise"]: - pw_i_bits_w = layer_config["pointwise"]["weight"]["integer_bits"] - pw_f_bits_w = layer_config["pointwise"]["weight"]["fractional_bits"] - if "bias" in layer_config: - pw_i_bits_b = layer_config["pointwise"]["bias"]["integer_bits"] - pw_f_bits_b = layer_config["pointwise"]["bias"]["fractional_bits"] - if "output" in layer_config: - if "quantize" in layer_config["output"]: - new_layer.quantize_output = layer_config["output"]["quantize"] - if "integer_bits" in layer_config["output"]: - i_output = layer_config["output"]["integer_bits"] - if "fractional_bits" in layer_config["output"]: - f_output = layer_config["output"]["fractional_bits"] + dw_i_weight = pw_i_weight = pw_i_bias = quant_params.default_weight_integer_bits + dw_f_weight = pw_f_weight = pw_f_bias = quant_params.default_weight_fractional_bits + i_input = i_output = quant_params.default_data_integer_bits + f_input = f_output = quant_params.default_data_fractional_bits + if layer_config is not None: + i_input, f_input = _get_quant_section( + layer_config.get("input"), i_input, f_input, new_layer.depthwise_conv, "quantize_input" + ) + depthwise_config = layer_config.get("depthwise", {}) + dw_i_weight, dw_f_weight = _get_quant_section(depthwise_config.get("weight"), dw_i_weight, dw_f_weight) + pointwise_config = layer_config.get("pointwise", {}) + pw_i_weight, pw_f_weight = _get_quant_section(pointwise_config.get("weight"), pw_i_weight, pw_f_weight) + pw_i_bias, pw_f_bias = _get_quant_section(pointwise_config.get("bias"), pw_i_bias, pw_f_bias) + i_output, f_output = _get_quant_section( + layer_config.get("output"), i_output, f_output, new_layer, "quantize_output" + ) new_layer.depthwise_conv.i_input = i_input new_layer.depthwise_conv.f_input = f_input - new_layer.depthwise_conv.i_weight = dw_i_bits_w - new_layer.depthwise_conv.f_weight = dw_f_bits_w - new_layer.pointwise_conv.i_weight = pw_i_bits_w - new_layer.pointwise_conv.f_weight = pw_f_bits_w - new_layer.pointwise_conv.i_bias = pw_i_bits_b - new_layer.pointwise_conv.f_bias = pw_f_bits_b + new_layer.depthwise_conv.i_weight = dw_i_weight + new_layer.depthwise_conv.f_weight = dw_f_weight + new_layer.pointwise_conv.i_weight = pw_i_weight + new_layer.pointwise_conv.f_weight = pw_f_weight + new_layer.pointwise_conv.i_bias = pw_i_bias + new_layer.pointwise_conv.f_bias = pw_f_bias new_layer.pointwise_conv.i_output = i_output new_layer.pointwise_conv.f_output = f_output else: - i_bits_w = i_bits_b = config.quantization_parameters.default_weight_integer_bits - f_bits_w = f_bits_b = config.quantization_parameters.default_weight_fractional_bits - if layer.name in layer_specific: - layer_config = layer_specific[layer.name] - if "input" in layer_config: - if "quantize" in layer_config["input"]: - new_layer.quantize_input = layer_config["input"]["quantize"] - if "integer_bits" in layer_config["input"]: - new_layer.i_input = layer_config["input"]["integer_bits"] - if "fractional_bits" in layer_config["input"]: - new_layer.f_input = layer_config["input"]["fractional_bits"] - if "weight" in layer_config: - i_bits_w = layer_config["weight"]["integer_bits"] - f_bits_w = layer_config["weight"]["fractional_bits"] - if "bias" in layer_config: - i_bits_b = layer_config["bias"]["integer_bits"] - f_bits_b = layer_config["bias"]["fractional_bits"] - if "output" in layer_config: - if "quantize" in layer_config["output"]: - new_layer.quantize_output = layer_config["output"]["quantize"] - if "integer_bits" in layer_config["output"]: - new_layer.i_output = layer_config["output"]["integer_bits"] - if "fractional_bits" in layer_config["output"]: - new_layer.f_output = layer_config["output"]["fractional_bits"] - new_layer.i_weight = i_bits_w - new_layer.f_weight = f_bits_w - new_layer.i_bias = i_bits_b - new_layer.f_bias = f_bits_b - new_layer.weight_quantizer.i_init = float(i_bits_w) - new_layer.weight_quantizer.f_init = float(f_bits_w) - new_layer.bias_quantizer.i_init = float(i_bits_b) - new_layer.bias_quantizer.f_init = float(f_bits_b) - - -def get_enable_pruning(layer, config): + i_weight = i_bias = quant_params.default_weight_integer_bits + f_weight = f_bias = quant_params.default_weight_fractional_bits + if layer_config is not None: + new_layer.i_input, new_layer.f_input = _get_quant_section( + layer_config.get("input"), new_layer.i_input, new_layer.f_input, new_layer, "quantize_input" + ) + i_weight, f_weight = _get_quant_section(layer_config.get("weight"), i_weight, f_weight) + i_bias, f_bias = _get_quant_section(layer_config.get("bias"), i_bias, f_bias) + new_layer.i_output, new_layer.f_output = _get_quant_section( + layer_config.get("output"), new_layer.i_output, new_layer.f_output, new_layer, "quantize_output" + ) + new_layer.i_weight = i_weight + new_layer.f_weight = f_weight + new_layer.i_bias = i_bias + new_layer.f_bias = f_bias + new_layer.weight_quantizer.i_init = float(i_weight) + new_layer.weight_quantizer.f_init = float(f_weight) + new_layer.bias_quantizer.i_init = float(i_bias) + new_layer.bias_quantizer.f_init = float(f_bias) + + +def _get_enable_pruning(layer, config): enable_pruning = config.pruning_parameters.enable_pruning if isinstance(layer, (SeparableConv2D, PQSeparableConv2d)): enable_pruning_depthwise = enable_pruning_pointwise = True @@ -2824,7 +2453,7 @@ def get_enable_pruning(layer, config): return enable_pruning -def populate_config_with_all_layers(model, config): +def _populate_config_with_all_layers(model, config): """Create a default config, where all the layers are added to the disable_pruning list, and have their own default quantization bits in layer_specific. By default input/output quantization is disabled. """ @@ -2917,7 +2546,7 @@ def post_training_prune(model, config, calibration_data): def get_ebops(model, **kwargs): ebops = 0 for m in model.layers: - if isinstance(m, (PQWeightBiasBase)): + if isinstance(m, PQWeightBiasBase): ebops += m.ebops(include_mask=m.enable_pruning) elif isinstance(m, (PQAvgPoolBase, PQBatchNormalization, PQActivation, PQSoftmax, PQMultiheadAttention)): ebops += m.ebops() diff --git a/src/pquant/core/keras/quantizer.py b/src/pquant/core/keras/quantizer.py index 396c8ac..c3f0405 100644 --- a/src/pquant/core/keras/quantizer.py +++ b/src/pquant/core/keras/quantizer.py @@ -1,11 +1,11 @@ -from enum import Enum - import keras from hgq.quantizer import Quantizer as HGQQuantizer from hgq.quantizer import QuantizerConfig from keras import ops from quantizers import get_fixed_quantizer +from pquant.core.constants import QuantizationGranularity + @keras.saving.register_keras_serializable(package="PQuantML") class Quantizer(keras.layers.Layer): @@ -19,7 +19,7 @@ def __init__( round_mode="RND", is_heterogeneous=False, is_data=False, - granularity="per_tensor", + granularity=QuantizationGranularity.PER_TENSOR, hgq_gamma=0, place="datalane", dynamic_data=True, @@ -35,7 +35,7 @@ def __init__( self.is_data = is_data self.dynamic_data = dynamic_data self.place = place - self.granularity = granularity.value if isinstance(granularity, Enum) else granularity + self.granularity = QuantizationGranularity(granularity).value self.quantizer = create_quantizer( self.k_init, self.i_init, @@ -53,9 +53,8 @@ def __init__( def calculate_bits_from_abs(self, abs_x): m = ops.ceil(ops.log(abs_x + 1e-6) / ops.log(2.0)) int_bits = ops.maximum(m, 0.0) - b = self.b if hasattr(self, "b") else self.b_init - int_bits = ops.minimum(m, b - self.k) - frac_bits = ops.maximum(b - int_bits - self.k_init, 0.0) + int_bits = ops.minimum(int_bits, self.b - self.k) + frac_bits = ops.maximum(self.b - int_bits - self.k, 0.0) return int_bits, frac_bits def compute_data_dynamic_bits(self, x): @@ -66,10 +65,10 @@ def compute_data_dynamic_bits(self, x): return self.calculate_bits_from_abs(abs_x) def compute_weight_dynamic_bits(self, x): - if self.granularity == "per_tensor": + if self.granularity == QuantizationGranularity.PER_TENSOR or ops.ndim(x) == 1: _, i, f = self.get_quantization_bits() return i, f - if self.granularity == "per_channel": + if self.granularity == QuantizationGranularity.PER_CHANNEL: if ops.ndim(x) == 2: abs_x = ops.max(ops.abs(x), axis=0, keepdims=True) elif ops.ndim(x) == 3: @@ -78,7 +77,7 @@ def compute_weight_dynamic_bits(self, x): abs_x = ops.max(ops.abs(x), axis=(0, 1, 2), keepdims=True) else: raise ValueError("Unsupported tensor rank") - elif self.granularity == "per_weight": + elif self.granularity == QuantizationGranularity.PER_WEIGHT: abs_x = ops.abs(x) else: raise ValueError(f"compute_dynamic_bits called for granularity={self.granularity}") @@ -89,40 +88,39 @@ def compute_dynamic_bits(self, x): return self.compute_data_dynamic_bits(x) return self.compute_weight_dynamic_bits(x) + def _dynamic_bits_shape(self, input_shape): + """Shape the dynamically computed i/f will have, without computing them (used by build).""" + if self.is_data or len(input_shape) == 1: + return () # data and bias are per_tensor, so scalar shape + if self.granularity == QuantizationGranularity.PER_CHANNEL: + if len(input_shape) not in (2, 3, 4): + raise ValueError("Unsupported tensor rank") + return (1,) * (len(input_shape) - 1) + (input_shape[-1],) + return tuple(input_shape) # per_weight + def build(self, input_shape): if self.use_hgq: shape = tuple(input_shape) if not self.is_data else (1,) + tuple(input_shape[1:]) - self.k = self.add_weight(shape=shape, initializer=keras.initializers.Constant(self.k_init), trainable=False) - self.i = self.add_weight(shape=shape, initializer=keras.initializers.Constant(self.i_init), trainable=False) - self.f = self.add_weight(shape=shape, initializer=keras.initializers.Constant(self.f_init), trainable=False) - self.b = self.add_weight( - shape=shape, - initializer=keras.initializers.Constant(self.k_init + self.i_init + self.f_init), - trainable=False, - ) if not self.quantizer.built: self.quantizer.build(shape) self.set_quantization_bits(self.i_init, self.f_init) - elif self.granularity == "per_tensor": - self.k = self.add_weight(shape=(), initializer=keras.initializers.Constant(self.k_init), trainable=False) - self.i = self.add_weight(shape=(), initializer=keras.initializers.Constant(self.i_init), trainable=False) - self.f = self.add_weight(shape=(), initializer=keras.initializers.Constant(self.f_init), trainable=False) - self.b = self.add_weight( - shape=(), initializer=keras.initializers.Constant(self.k_init + self.i_init + self.f_init), trainable=False - ) + elif self.granularity == QuantizationGranularity.PER_TENSOR: + self._build_params(shape=()) else: - i, _ = self.compute_dynamic_bits(keras.ops.ones(input_shape)) - self.k = self.add_weight(shape=i.shape, initializer=keras.initializers.Constant(self.k_init), trainable=False) - self.i = self.add_weight(shape=i.shape, initializer=keras.initializers.Constant(self.i_init), trainable=False) - self.f = self.add_weight(shape=i.shape, initializer=keras.initializers.Constant(self.f_init), trainable=False) - self.b = self.add_weight( - shape=i.shape, - initializer=keras.initializers.Constant(self.k_init + self.i_init + self.f_init), - trainable=False, - ) - + shape = self._dynamic_bits_shape(input_shape) + self._build_params(shape) super().build(input_shape) + def _build_params(self, shape): + self.k = self.add_weight(shape=shape, initializer=keras.initializers.Constant(self.k_init), trainable=False) + self.i = self.add_weight(shape=shape, initializer=keras.initializers.Constant(self.i_init), trainable=False) + self.f = self.add_weight(shape=shape, initializer=keras.initializers.Constant(self.f_init), trainable=False) + self.b = self.add_weight( + shape=shape, + initializer=keras.initializers.Constant(self.k_init + self.i_init + self.f_init), + trainable=False, + ) + def get_total_bits(self, shape): if self.use_hgq: return self.quantizer.bits_(shape) @@ -133,22 +131,29 @@ def get_total_bits(self, shape): def get_quantization_bits(self): if self.use_hgq: return self.quantizer.quantizer.k, self.quantizer.quantizer.i, self.quantizer.quantizer.f + if not hasattr(self, "i"): + return self.k_init, self.i_init, self.f_init return self.k, self.i, self.f def set_quantization_bits(self, i, f): if self.use_hgq: self.quantizer.quantizer._i.assign(self.quantizer.quantizer._i * 0.0 + i) self.quantizer.quantizer._f.assign(self.quantizer.quantizer._f * 0.0 + f) - self.i = i - self.f = f + elif hasattr(self, "i") and hasattr(self.i, "assign"): + self.i.assign(ops.zeros_like(self.i) + i) + self.f.assign(ops.zeros_like(self.f) + f) + self.b.assign(self.k + self.i + self.f) + else: + self.i = i + self.f = f def apply_final_compression(self): - if self.use_hgq and not self.quantizer.built or not self.built: + if self.use_hgq or not self.built: return k, i, f = self.get_quantization_bits() - self.i.assign(i) - self.f.assign(f) - self.b.assign(k + i + f) + self.i.assign(ops.zeros_like(self.i) + i) + self.f.assign(ops.zeros_like(self.f) + f) + self.b.assign(ops.zeros_like(self.b) + k + i + f) self.final_compression_done = True def post_pre_train_function(self): @@ -159,10 +164,9 @@ def call(self, x, training=None): return self.quantizer(x, training=training) if not training: return self.quantizer(x, k=self.k, i=self.i, f=self.f, training=training) - else: - i, f = self.compute_dynamic_bits(x) - self.i.assign(i) - self.f.assign(f) + i, f = self.compute_dynamic_bits(x) + self.i.assign(i) + self.f.assign(f) return self.quantizer(x, k=self.k, i=i, f=f, training=training) def hgq_loss(self): @@ -224,11 +228,11 @@ def axis_kwargs_for_granularity(granularity, is_data): per_channel is intentionally NOT supported for HGQ (the channel axis is layout-dependent, so we don't guess it). """ - if granularity == "per_tensor": + if granularity == QuantizationGranularity.PER_TENSOR: return {"heterogeneous_axis": ()} - if granularity == "per_weight": + if granularity == QuantizationGranularity.PER_WEIGHT: return {"homogeneous_axis": (0,) if is_data else ()} - if granularity == "per_channel": + if granularity == QuantizationGranularity.PER_CHANNEL: raise ValueError("per_channel granularity is not supported for HGQ. Use 'per_tensor' or 'per_weight'.") raise ValueError(f"Unsupported granularity: {granularity}") @@ -262,7 +266,16 @@ def create_hgq_data_quantizer(k, i, f, overflow, round_mode, axis_kwargs, gamma= def create_quantizer( - k, i, f, overflow, round_mode, is_heterogeneous, is_data, place="datalane", granularity="per_weight", gamma=1e-8 + k, + i, + f, + overflow, + round_mode, + is_heterogeneous, + is_data, + place="datalane", + granularity=QuantizationGranularity.PER_WEIGHT, + gamma=1e-8, ): if is_heterogeneous: axis_kwargs = axis_kwargs_for_granularity(granularity, is_data) diff --git a/src/pquant/core/torch/fit_compress.py b/src/pquant/core/torch/fit_compress.py index d99e4fb..762863c 100644 --- a/src/pquant/core/torch/fit_compress.py +++ b/src/pquant/core/torch/fit_compress.py @@ -174,7 +174,7 @@ def print_info_bits(model): config, ) # Now add the layer specific configuration to the model - # add_layer_specific_quantization_to_model(trained_uncompressed_model, config) + # _add_layer_specific_quantization_to_model(trained_uncompressed_model, config) logging.info("Layerwise quantization bits after FITcompress : ", config.quantization_parameters.layer_specific) @@ -799,10 +799,6 @@ def astar(self, config): self.assign_parameters(self.model, params_quantized_unpruned) - self.post_fitcompress_calibration( - p_node.extract_config_from_node(self.layer_names)['quant_config'], config - ) - return ( p_node, p_node.extract_config_from_node(self.layer_names), diff --git a/src/pquant/core/torch/fixed_point_quantizer.py b/src/pquant/core/torch/fixed_point_quantizer.py index e3ecc4e..7ed3bd5 100644 --- a/src/pquant/core/torch/fixed_point_quantizer.py +++ b/src/pquant/core/torch/fixed_point_quantizer.py @@ -186,7 +186,7 @@ def forward(self, x, k, i, f, training=False): def forward_wrap_sm(self, x, k, i, f, training=False): def quant_fn(x): - return self.round(x, f, training and self.stochastic) + return self.round(x, f) x = wrap_sm_fn(x, k, i, f, training, quant_fn) return x diff --git a/src/pquant/core/torch/hgq_quantizer.py b/src/pquant/core/torch/hgq_quantizer.py index 26a468c..e62c9a0 100644 --- a/src/pquant/core/torch/hgq_quantizer.py +++ b/src/pquant/core/torch/hgq_quantizer.py @@ -15,6 +15,7 @@ import torch import torch.nn as nn +from pquant.core.constants import QuantizationGranularity from pquant.core.torch.fixed_point_quantizer import get_fixed_quantizer, round_conv logger = logging.getLogger(__name__) @@ -77,7 +78,7 @@ def __init__( overflow_mode: str, round_mode: str, is_data: bool, - granularity: str = "per_weight", + granularity: str = QuantizationGranularity.PER_WEIGHT, gamma: float = 1e-8, i_decay_speed: float = float("inf"), i_min: float = -23.0, @@ -97,7 +98,7 @@ def __init__( self.overflow_mode = overflow_mode.upper() self.round_mode = round_mode.upper() self.is_data = is_data - self.granularity = granularity + self.granularity = QuantizationGranularity(granularity).value self.gamma = gamma self.i_decay_speed = i_decay_speed self.i_min = i_min @@ -157,11 +158,11 @@ def _homogeneous_axis(self, ndim: int) -> tuple[int, ...]: HGQ supports only per_tensor and per_weight (data always shares the batch axis 0). per_channel is intentionally unsupported (the channel axis is layout-dependent). """ - if self.granularity == "per_tensor": + if self.granularity == QuantizationGranularity.PER_TENSOR: return tuple(range(ndim)) - if self.granularity == "per_weight": + if self.granularity == QuantizationGranularity.PER_WEIGHT: return (0,) if self.is_data else () - if self.granularity == "per_channel": + if self.granularity == QuantizationGranularity.PER_CHANNEL: raise ValueError("per_channel granularity is not supported for HGQ. Use 'per_tensor' or 'per_weight'.") raise ValueError(f"Unsupported granularity: {self.granularity}") diff --git a/src/pquant/core/torch/layers.py b/src/pquant/core/torch/layers.py index a6a969e..85e302f 100644 --- a/src/pquant/core/torch/layers.py +++ b/src/pquant/core/torch/layers.py @@ -1,10 +1,10 @@ +import math import typing from typing import Optional, Tuple, TypeVar, Union import torch import torch.nn as nn import torch.nn.functional as F -from torch.fx import symbolic_trace from torch.nn.common_types import _size_1_t, _size_2_t from pquant.core.torch.activations import PQActivation, PQSoftmax @@ -14,11 +14,33 @@ if typing.TYPE_CHECKING: from pquant.core.torch.fit_compress import call_fitcompress # noqa: 401 -from keras import ops - T = TypeVar("T") +def _resolve_data_quant_bits(quant_bits, config): + """Return (k, i, f) from an explicit tuple, or the config's data-lane defaults.""" + if quant_bits is not None: + return quant_bits + parameters = config.quantization_parameters + return ( + parameters.default_data_keep_negatives, + parameters.default_data_integer_bits, + parameters.default_data_fractional_bits, + ) + + +def _resolve_weight_quant_bits(quant_bits, config): + """Return (k, i, f) from an explicit tuple, or the config's weight defaults.""" + if quant_bits is not None: + return quant_bits + parameters = config.quantization_parameters + return ( + parameters.default_weight_keep_negatives, + parameters.default_weight_integer_bits, + parameters.default_weight_fractional_bits, + ) + + class PQWeightBiasBase(nn.Module): def __init__( self, @@ -40,32 +62,10 @@ def __init__( ): super().__init__(**kwargs) - if in_quant_bits is not None: - self.k_input, self.i_input, self.f_input = in_quant_bits - else: - self.k_input = config.quantization_parameters.default_data_keep_negatives - self.i_input = config.quantization_parameters.default_data_integer_bits - self.f_input = config.quantization_parameters.default_data_fractional_bits - - if weight_quant_bits is not None: - self.k_weight, self.i_weight, self.f_weight = weight_quant_bits - else: - self.k_weight = config.quantization_parameters.default_weight_keep_negatives - self.i_weight = config.quantization_parameters.default_weight_integer_bits - self.f_weight = config.quantization_parameters.default_weight_fractional_bits - if bias_quant_bits is not None: - self.k_bias, self.i_bias, self.f_bias = bias_quant_bits - else: - self.k_bias = config.quantization_parameters.default_weight_keep_negatives - self.i_bias = config.quantization_parameters.default_weight_integer_bits - self.f_bias = config.quantization_parameters.default_weight_fractional_bits - - if out_quant_bits is not None: - self.k_output, self.i_output, self.f_output = out_quant_bits - else: - self.k_output = config.quantization_parameters.default_data_keep_negatives - self.i_output = config.quantization_parameters.default_data_integer_bits - self.f_output = config.quantization_parameters.default_data_fractional_bits + self.k_input, self.i_input, self.f_input = _resolve_data_quant_bits(in_quant_bits, config) + self.k_weight, self.i_weight, self.f_weight = _resolve_weight_quant_bits(weight_quant_bits, config) + self.k_bias, self.i_bias, self.f_bias = _resolve_weight_quant_bits(bias_quant_bits, config) + self.k_output, self.i_output, self.f_output = _resolve_data_quant_bits(out_quant_bits, config) self.pruning_layer = get_pruning_layer(config=config, layer_type=layer_type) self.pruning_method = config.pruning_parameters.pruning_method @@ -88,7 +88,7 @@ def __init__( self.in_quant_granularity = in_quant_granularity if in_quant_granularity is not None else self.granularity self.bias_quant_granularity = bias_quant_granularity if bias_quant_granularity is not None else self.granularity self.out_quant_granularity = out_quant_granularity if out_quant_granularity is not None else self.granularity - self.final_compression_done = False + self.register_buffer("final_compression_done", torch.tensor(False)) self.built = False self.parallelization_factor = -1 self.hgq_beta = config.quantization_parameters.hgq_beta @@ -99,65 +99,67 @@ def __init__( self.saved_outputs = [] self.config = config - def check_is_built(self, input_shape): + def _check_is_built(self, input_shape): if self.built: return - # Build function to delay quantizer creation until after custom i,f bits have been set + # Quantizer creation is delayed until the first forward so custom i/f bits set + # after __init__ are picked up. if self.quantize_input: self.input_quantizer = Quantizer( - k=torch.tensor(self.k_input), - i=torch.tensor(self.i_input), - f=torch.tensor(self.f_input), + k=self.k_input, + i=self.i_input, + f=self.f_input, overflow=self.overflow_mode_data, round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=True, + granularity=self.in_quant_granularity, hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, - granularity=self.in_quant_granularity, ) self.weight_quantizer = Quantizer( - k=torch.tensor(self.k_weight), - i=torch.tensor(self.i_weight), - f=torch.tensor(self.f_weight), + k=self.k_weight, + i=self.i_weight, + f=self.f_weight, overflow=self.overflow_mode_parameters, round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=False, - hgq_gamma=self.hgq_gamma, granularity=self.weight_quant_granularity, + hgq_gamma=self.hgq_gamma, place="weight", + shape=self._weight.shape, ) - self.bias_quantizer = Quantizer( - k=torch.tensor(self.k_bias), - i=torch.tensor(self.i_bias), - f=torch.tensor(self.f_bias), + k=self.k_bias, + i=self.i_bias, + f=self.f_bias, overflow=self.overflow_mode_parameters, round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=False, - hgq_gamma=self.hgq_gamma, granularity=self.bias_quant_granularity, + hgq_gamma=self.hgq_gamma, place="bias", + shape=None if self._bias is None else self._bias.shape, ) if self.quantize_output: self.output_quantizer = Quantizer( - k=torch.tensor(self.k_output), - i=torch.tensor(self.i_output), - f=torch.tensor(self.f_output), + k=self.k_output, + i=self.i_output, + f=self.f_output, overflow=self.overflow_mode_data, round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=True, + granularity=self.out_quant_granularity, hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, - granularity=self.out_quant_granularity, ) - self.n_parallel = ops.prod(tuple(input_shape)[1:-1]) + self.n_parallel = math.prod(tuple(input_shape)[1:-1]) self.parallelization_factor = self.parallelization_factor if self.parallelization_factor > 0 else self.n_parallel self.built = True self.input_shape = (1,) + input_shape[1:] @@ -177,15 +179,27 @@ def get_output_quantization_bits(self): def apply_final_compression(self): pass + def _register_compressed_parameters(self, bias): + """Store the wrapped layer's weight/bias as `_weight`/`_bias`; the `weight`/`bias` + properties then return their pruned and quantized views.""" + self._weight = nn.Parameter(self.weight.clone()).to(self.weight.device) + self.register_parameter("_weight", self._weight) + if bias: + self._bias = nn.Parameter(self.bias.clone()).to(self.bias.device) + self.register_parameter("_bias", self._bias) + else: + self.register_parameter("_bias", None) + self.pruning_layer.build(self._weight.shape) + def post_pre_train_function(self): self.is_pretraining = False if self.pruning_layer is not None: self.pruning_layer.post_pre_train_function() - def save_weights(self): + def _save_weights(self): self.init_weight = self._weight.clone() - def rewind_weights(self): + def _rewind_weights(self): if not hasattr(self, "init_weight"): return self._weight.data = self.init_weight.clone() @@ -193,6 +207,14 @@ def rewind_weights(self): def ebops(self): return 0.0 + def _masked_weight_bits(self, bw_ker): + """Zero the bit counts of weights that are pruned away or below the quantization step size.""" + bw_ker = bw_ker * self.pruning_layer.get_hard_mask() + _, _, f = self.get_weight_quantization_bits() + quantization_step_size = 2 ** (-f - 1) + step_size_mask = (torch.abs(self._weight) > quantization_step_size).float() + return bw_ker * step_size_mask + def hgq_loss(self): if self.is_pretraining or not self.use_hgq: return 0.0 @@ -207,20 +229,20 @@ def hgq_loss(self): return loss def quantize(self, x, quantizer): - if self.enable_quantization and not self.is_fitcompress_pretraining(): + if self.enable_quantization and not self._is_fitcompress_pretraining(): return quantizer(x) if x is not None else x return x - def prune(self, weight): + def _prune(self, weight): if self.enable_pruning: weight = self.pruning_layer(weight) return weight - def is_fitcompress_pretraining(self): + def _is_fitcompress_pretraining(self): return self.is_pretraining and self.use_fitcompress def pre_forward(self, x): - self.check_is_built(x.shape) + self._check_is_built(x.shape) if self.post_fitcompress_calibration: self.saved_inputs.append(x) return x @@ -230,7 +252,7 @@ def pre_forward(self, x): self.pruning_layer.collect_input(x, self.weight, self.training) return x - def post_forward(self, x): + def _post_forward(self, x): if self.post_fitcompress_calibration: self.saved_outputs.append(x) return x @@ -287,47 +309,35 @@ def __init__( self.in_features = in_features self.out_features = out_features self.use_fitcompress = config.fitcompress_parameters.enable_fitcompress - self._weight = nn.Parameter(self.weight.clone()).to(self.weight.device) - self.register_parameter("_weight", self._weight) - if bias: - self._bias = nn.Parameter(self.bias.clone()).to(self.bias.device) - self.register_parameter("_bias", self._bias) - else: - self.register_parameter("_bias", None) - self.pruning_layer.build(self._weight.shape) - self.final_compression_done = nn.Parameter(torch.tensor(False), requires_grad=False) + self._register_compressed_parameters(bias) def ebops(self, include_mask=False): bw_inp = self.input_quantizer.get_total_bits(self.input_shape) - bw_ker = self.weight_quantizer.get_total_bits(ops.shape(self._weight)) + bw_ker = self.weight_quantizer.get_total_bits(self._weight.shape) if include_mask: - bw_ker = bw_ker * self.pruning_layer.get_hard_mask() - _, _, f = self.get_weight_quantization_bits() - quantization_step_size = 2 ** (-f - 1) - step_size_mask = (torch.abs(self._weight) >= quantization_step_size).float() - bw_ker = bw_ker * step_size_mask - ebops = ops.sum(F.linear(bw_inp, bw_ker)) + bw_ker = self._masked_weight_bits(bw_ker) + ebops = torch.sum(F.linear(bw_inp, bw_ker)) if self._bias is not None: - bw_bias = self.bias_quantizer.get_total_bits(ops.shape(self._bias)) - size = ops.cast(ops.prod(self.input_shape[:-1]) * self.out_features, self._weight.dtype) - ebops += ops.mean(bw_bias) * size + bw_bias = self.bias_quantizer.get_total_bits(self._bias.shape) + size = float(math.prod(self.input_shape[:-1]) * self.out_features) + ebops += torch.mean(bw_bias) * size ebops = ebops * self.parallelization_factor / self.n_parallel return ebops @property def weight(self): - if self.final_compression_done or self.is_fitcompress_pretraining(): + if self.final_compression_done or self._is_fitcompress_pretraining(): return self._weight if self.pruning_first: - weight = self.prune(self._weight) + weight = self._prune(self._weight) return self.quantize(weight, self.weight_quantizer) else: weight = self.quantize(self._weight, self.weight_quantizer) - return self.prune(weight) + return self._prune(weight) @property def bias(self): - if self.final_compression_done or self.is_fitcompress_pretraining(): + if self.final_compression_done or self._is_fitcompress_pretraining(): return self._bias bias = self.quantize(self._bias, self.bias_quantizer) return bias @@ -336,12 +346,12 @@ def apply_final_compression(self): self._weight.data = self.weight if self._bias is not None: self._bias.data = self.bias - self.final_compression_done.data = torch.tensor(True) + self.final_compression_done.fill_(True) def forward(self, x): x = self.pre_forward(x) x = super().forward(x) - x = self.post_forward(x) + x = self._post_forward(x) return x def extra_repr(self) -> str: @@ -357,7 +367,79 @@ def extra_repr(self) -> str: ) -class PQConv2d(PQWeightBiasBase, nn.Conv2d): +class PQConvBase(PQWeightBiasBase): + """Pruning/quantization behavior shared by PQConv1d and PQConv2d.""" + + conv_bits_fn = None # F.conv1d / F.conv2d, set by the subclasses + + def ebops(self, include_mask=False): + bw_inp = self.input_quantizer.get_total_bits(self.input_shape) + bw_ker = self.weight_quantizer.get_total_bits(self._weight.shape) + if include_mask: + bw_ker = self._masked_weight_bits(bw_ker) + if self.parallelization_factor < 0: + ebops = torch.sum( + self.conv_bits_fn(bw_inp, bw_ker, stride=self.stride, padding=self.padding, dilation=self.dilation) + ) + else: + spatial_axes = tuple(range(2, 2 + len(self.kernel_size))) + bw_inp = torch.amax(bw_inp, dim=(0,) + spatial_axes) + bw_ker = torch.sum(bw_ker, dim=spatial_axes) + ebops = torch.sum(bw_inp[None, :] * bw_ker) + if self._bias is not None: + size = float(math.prod(self.input_shape)) + bw_bias = self.bias_quantizer.get_total_bits(self._bias.shape) + ebops += torch.mean(bw_bias) * size + return ebops + + @property + def weight(self): + if self.final_compression_done: + return self._weight + if self.pruning_first: + weight = self._prune(self._weight) + return self.quantize(weight, self.weight_quantizer) + weight = self.quantize(self._weight, self.weight_quantizer) + return self._prune(weight) + + @property + def bias(self): + if self.final_compression_done: + return self._bias + return self.quantize(self._bias, self.bias_quantizer) + + def apply_final_compression(self): + self._weight.data = self.weight + if self._bias is not None: + self._bias.data = self.bias + self.final_compression_done.fill_(True) + + def forward(self, x): + x = self.pre_forward(x) + x = super().forward(x) + x = self._post_forward(x) + return x + + def extra_repr(self): + s = "{in_channels}, {out_channels}, kernel_size={kernel_size}, stride={stride}" + if self.padding != (0,) * len(self.padding): + s += ", padding={padding}" + if self.dilation != (1,) * len(self.dilation): + s += ", dilation={dilation}" + if self.output_padding != (0,) * len(self.output_padding): + s += ", output_padding={output_padding}" + if self.groups != 1: + s += ", groups={groups}" + if self._bias is None: + s += ", bias=False" + if self.padding_mode != "zeros": + s += ", padding_mode={padding_mode}" + s += ", quantize_input={quantize_input}" + s += ", quantize_output={quantize_output}" + return s.format(**self.__dict__) + + +class PQConv2d(PQConvBase, nn.Conv2d): def __init__( self, config, @@ -413,100 +495,12 @@ def __init__( **kwargs, ) self.use_fitcompress = config.fitcompress_parameters.enable_fitcompress - self._weight = nn.Parameter(self.weight.clone()).to(self.weight.device) - self.register_parameter("_weight", self._weight) - if bias: - self._bias = nn.Parameter(self.bias.clone()).to(self.bias.device) - self.register_parameter("_bias", self._bias) - else: - self.register_parameter("_bias", None) - self.pruning_layer.build(self._weight.shape) - - def ebops(self, include_mask=False): - bw_inp = self.input_quantizer.get_total_bits(self.input_shape) - bw_ker = self.weight_quantizer.get_total_bits(ops.shape(self._weight)) - if include_mask: - bw_ker = bw_ker * self.pruning_layer.get_hard_mask() - _, _, f = self.get_weight_quantization_bits() - quantization_step_size = 2 ** (-f - 1) - step_size_mask = (torch.abs(self._weight) > quantization_step_size).float() - bw_ker = bw_ker * step_size_mask - if self.parallelization_factor < 0: - ebops = ops.sum(F.conv2d(bw_inp, bw_ker, stride=self.stride, padding=self.padding, dilation=self.dilation)) - else: - reduce_axis_kernel = tuple(range(2, 4)) - reduce_axis_input = (0,) + tuple(range(2, 4)) - - bw_inp = ops.max(bw_inp, axis=reduce_axis_input) - bw_ker = ops.sum(bw_ker, axis=reduce_axis_kernel) - ebops = ops.sum(bw_inp[None, :] * bw_ker) - if self._bias is not None: - size = ops.cast(ops.prod(list(self.input_shape)), self.weight.dtype) - bw_bias = self.bias_quantizer.get_total_bits(ops.shape(self._bias)) - ebops += ops.mean(bw_bias) * size - return ebops - - @property - def weight(self): - if self.final_compression_done: - return self._weight - if self.pruning_first: - weight = self.prune(self._weight) - return self.quantize(weight, self.weight_quantizer) - else: - weight = self.quantize(self._weight, self.weight_quantizer) - return self.prune(weight) - - @property - def bias(self): - if self.final_compression_done: - return self._bias - bias = self.quantize(self._bias, self.bias_quantizer) - return bias - - def apply_final_compression(self): - self._weight.data = self.weight - if self._bias is not None: - self._bias.data = self.bias - self.final_compression_done = True - - def forward(self, x): - x = self.pre_forward(x) - weight = self.weight - bias = self.bias - x = F.conv2d( - x, - weight, - bias, - self.stride, - self.padding, - self.dilation, - self.groups, - ) - x = self.post_forward(x) - return x - - def extra_repr(self): - s = "{in_channels}, {out_channels}, kernel_size={kernel_size}, stride={stride}" - if self.padding != (0,) * len(self.padding): - s += ", padding={padding}" - if self.dilation != (1,) * len(self.dilation): - s += ", dilation={dilation}" - if self.output_padding != (0,) * len(self.output_padding): - s += ", output_padding={output_padding}" - if self.groups != 1: - s += ", groups={groups}" - if self._bias is None: - s += ", bias=False" - if self.padding_mode != "zeros": - s += ", padding_mode={padding_mode}" - s += ", self.quantize_input={quantize_input} " - s += ", self.quantize_output={quantize_output}" + self._register_compressed_parameters(bias) - return s.format(**self.__dict__) + conv_bits_fn = staticmethod(F.conv2d) -class PQConv1d(PQWeightBiasBase, nn.Conv1d): +class PQConv1d(PQConvBase, nn.Conv1d): def __init__( self, config, @@ -562,101 +556,24 @@ def __init__( **kwargs, ) self.use_fitcompress = config.fitcompress_parameters.enable_fitcompress - self._weight = nn.Parameter(self.weight.clone()).to(self.weight.device) - self.register_parameter("_weight", self._weight) - if bias: - self._bias = nn.Parameter(self.bias.clone()).to(self.bias.device) - self.register_parameter("_bias", self._bias) - else: - self.register_parameter("_bias", None) - self.pruning_layer.build(self._weight.shape) - - def ebops(self, include_mask=False): - bw_inp = self.input_quantizer.get_total_bits(self.input_shape) - bw_ker = self.weight_quantizer.get_total_bits(ops.shape(self._weight)) - if include_mask: - bw_ker = bw_ker * self.pruning_layer.get_hard_mask() - _, _, f = self.get_weight_quantization_bits() - quantization_step_size = 2 ** (-f - 1) - step_size_mask = (torch.abs(self._weight) > quantization_step_size).float() - bw_ker = bw_ker * step_size_mask - if self.parallelization_factor < 0: - ebops = ops.sum(F.conv1d(bw_inp, bw_ker, stride=self.stride, padding=self.padding, dilation=self.dilation)) - else: - reduce_axis_kernel = tuple(range(2, 3)) - reduce_axis_input = (0,) + tuple(range(2, 3)) - - bw_inp = ops.max(bw_inp, axis=reduce_axis_input) - bw_ker = ops.sum(bw_ker, axis=reduce_axis_kernel) - ebops = ops.sum(bw_inp[None, :] * bw_ker) - if self.bias is not None: - size = ops.cast(ops.prod(list(self.input_shape)), self.weight.dtype) - bw_bias = self.bias_quantizer.get_total_bits(ops.shape(self._bias)) - ebops += ops.mean(bw_bias) * size - return ebops - - @property - def weight(self): - if self.final_compression_done: - return self._weight - if self.pruning_first: - weight = self.prune(self._weight) - return self.quantize(weight, self.weight_quantizer) - else: - weight = self.quantize(self._weight, self.weight_quantizer) - return self.prune(weight) - - @property - def bias(self): - if self.final_compression_done: - return self._bias - bias = self.quantize(self._bias, self.bias_quantizer) - return bias + self._register_compressed_parameters(bias) - def apply_final_compression(self): - self._weight.data = self.weight - if self._bias is not None: - self._bias.data = self.bias - self.final_compression_done = True - - def forward(self, x): - x = self.pre_forward(x) - x = super().forward(x) - x = self.post_forward(x) - return x - - def extra_repr(self): - s = "{in_channels}, {out_channels}, kernel_size={kernel_size}, stride={stride}" - if self.padding != (0,) * len(self.padding): - s += ", padding={padding}" - if self.dilation != (1,) * len(self.dilation): - s += ", dilation={dilation}" - if self.output_padding != (0,) * len(self.output_padding): - s += ", output_padding={output_padding}" - if self.groups != 1: - s += ", groups={groups}" - if self._bias is None: - s += ", bias=False" - if self.padding_mode != "zeros": - s += ", padding_mode={padding_mode}" - s += ", self.quantize_input={quantize_input}" - s += ", self.quantize_output={quantize_output}" - return s.format(**self.__dict__) + conv_bits_fn = staticmethod(F.conv1d) def add_compression_layers(model, config, input_shape=None, add_missing_quantizers=False): - model = add_quantized_activations_to_model_layer(model, config) - model = add_pruning_to_model(model, config) + device = next((p.device for p in model.parameters()), torch.device("cpu")) + model = _add_quantized_activations_to_model_layer(model, config) + model = _add_pruning_to_model(model, config) if add_missing_quantizers: # Imported here (not at module top) to avoid a circular import: tracing.py # imports the layer classes defined in this module. from pquant.core.torch.tracing import check_quantization model = check_quantization(model, add_missing_quantizers=True, config=config) - model.to("cuda") + model.to(device) if input_shape is not None: - model(torch.rand(input_shape).to("cuda")) - model.to("cuda") + model(torch.rand(input_shape).to(device)) return model @@ -673,19 +590,8 @@ def __init__( **kwargs, ): super().__init__(**kwargs) - if in_quant_bits is not None: - self.k_input, self.i_input, self.f_input = in_quant_bits - else: - self.k_input = config.quantization_parameters.default_data_keep_negatives - self.i_input = config.quantization_parameters.default_data_integer_bits - self.f_input = config.quantization_parameters.default_data_fractional_bits - - if out_quant_bits is not None: - self.k_output, self.i_output, self.f_output = out_quant_bits - else: - self.k_output = config.quantization_parameters.default_data_keep_negatives - self.i_output = config.quantization_parameters.default_data_integer_bits - self.f_output = config.quantization_parameters.default_data_fractional_bits + self.k_input, self.i_input, self.f_input = _resolve_data_quant_bits(in_quant_bits, config) + self.k_output, self.i_output, self.f_output = _resolve_data_quant_bits(out_quant_bits, config) self.overflow_mode_data = config.quantization_parameters.overflow_mode_data self.config = config self.is_pretraining = True @@ -706,30 +612,30 @@ def __init__( def build(self, input_shape): self.input_quantizer = Quantizer( - k=torch.tensor(self.k_input), - i=torch.tensor(self.i_input), - f=torch.tensor(self.f_input), + k=self.k_input, + i=self.i_input, + f=self.f_input, overflow=self.overflow_mode_data, round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=True, + granularity=self.in_quant_granularity, hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, - granularity=self.in_quant_granularity, ) self.output_quantizer = Quantizer( - k=torch.tensor(self.k_output), - i=torch.tensor(self.i_output), - f=torch.tensor(self.f_output), + k=self.k_output, + i=self.i_output, + f=self.f_output, overflow=self.overflow_mode_data, round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=True, + granularity=self.out_quant_granularity, hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, - granularity=self.out_quant_granularity, ) self.input_shape = (1,) + input_shape[1:] @@ -756,13 +662,13 @@ def hgq_loss(self): loss += self.output_quantizer.hgq_loss() return loss - def is_fitcompress_pretraining(self): + def _is_fitcompress_pretraining(self): return self.is_pretraining and self.use_fitcompress - def pre_pooling(self, x): + def _pre_pooling(self, x): if not hasattr(self, "input_quantizer"): self.build(x.shape) - if self.is_fitcompress_pretraining(): + if self._is_fitcompress_pretraining(): if self.post_fitcompress_calibration: # Save inputs self.saved_inputs.append(x) @@ -772,8 +678,8 @@ def pre_pooling(self, x): x = self.input_quantizer(x) return x - def post_pooling(self, x): - if self.quantize_output and self.enable_quantization and not self.is_fitcompress_pretraining(): + def _post_pooling(self, x): + if self.quantize_output and self.enable_quantization and not self._is_fitcompress_pretraining(): x = self.output_quantizer(x) return x @@ -815,9 +721,9 @@ def __init__( ) def forward(self, x): - x = self.pre_pooling(x) + x = self._pre_pooling(x) x = super().forward(x) - x = self.post_pooling(x) + x = self._post_pooling(x) return x @@ -857,51 +763,29 @@ def __init__( ) def forward(self, x): - x = self.pre_pooling(x) + x = self._pre_pooling(x) x = super().forward(x) - x = self.post_pooling(x) + x = self._post_pooling(x) return x -class PQBatchNorm2d(nn.BatchNorm2d): - def __init__( +class PQBatchNormBase: + """Quantization behavior shared by PQBatchNorm1d and PQBatchNorm2d.""" + + def _init_quantization( self, config, - num_features: int, - eps: float = 1e-5, - momentum: typing.Optional[float] = 0.1, - affine: bool = True, - track_running_stats: bool = True, - device=None, - dtype=None, - quantize_input=True, - in_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, - in_quant_granularity=None, - weight_quant_granularity=None, - bias_quant_granularity=None, + quantize_input, + in_quant_bits, + weight_quant_bits, + bias_quant_bits, + in_quant_granularity, + weight_quant_granularity, + bias_quant_granularity, ): - super().__init__(num_features, eps, momentum, affine, track_running_stats, device=device, dtype=dtype) - if in_quant_bits is not None: - self.k_input, self.i_input, self.f_input = in_quant_bits - else: - self.k_input = config.quantization_parameters.default_data_keep_negatives - self.i_input = config.quantization_parameters.default_data_integer_bits - self.f_input = config.quantization_parameters.default_data_fractional_bits - - if weight_quant_bits is not None: - self.k_weight, self.i_weight, self.f_weight = weight_quant_bits - else: - self.k_weight = config.quantization_parameters.default_weight_keep_negatives - self.i_weight = config.quantization_parameters.default_weight_integer_bits - self.f_weight = config.quantization_parameters.default_weight_fractional_bits - if bias_quant_bits is not None: - self.k_bias, self.i_bias, self.f_bias = bias_quant_bits - else: - self.k_bias = config.quantization_parameters.default_weight_keep_negatives - self.i_bias = config.quantization_parameters.default_weight_integer_bits - self.f_bias = config.quantization_parameters.default_weight_fractional_bits + self.k_input, self.i_input, self.f_input = _resolve_data_quant_bits(in_quant_bits, config) + self.k_weight, self.i_weight, self.f_weight = _resolve_weight_quant_bits(weight_quant_bits, config) + self.k_bias, self.i_bias, self.f_bias = _resolve_weight_quant_bits(bias_quant_bits, config) self.overflow_mode_parameters = config.quantization_parameters.overflow_mode_parameters self.overflow_mode_data = config.quantization_parameters.overflow_mode_data self.round_mode = config.quantization_parameters.round_mode @@ -924,49 +808,51 @@ def __init__( else: self.register_parameter("_bias", None) self.built = False - self.final_compression_done = False + self.register_buffer("final_compression_done", torch.tensor(False)) self.is_pretraining = True self.post_fitcompress_calibration = False self.saved_inputs = [] - def check_is_built(self, input_shape): + def _check_is_built(self, input_shape): if self.built: return self.built = True self.input_quantizer = Quantizer( - k=torch.tensor(self.k_input), - i=torch.tensor(self.i_input), - f=torch.tensor(self.f_input), + k=self.k_input, + i=self.i_input, + f=self.f_input, overflow=self.overflow_mode_data, round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=True, + granularity=self.in_quant_granularity, hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, - granularity=self.in_quant_granularity, ) self.weight_quantizer = Quantizer( - k=torch.tensor(self.k_weight), - i=torch.tensor(self.i_weight), - f=torch.tensor(self.f_weight), - round_mode=self.round_mode, + k=self.k_weight, + i=self.i_weight, + f=self.f_weight, overflow=self.overflow_mode_parameters, - is_data=False, + round_mode=self.round_mode, is_heterogeneous=self.use_hgq, - place="weight", + is_data=False, granularity=self.weight_quant_granularity, + place="weight", + shape=self._weight.shape, ) self.bias_quantizer = Quantizer( - k=torch.tensor(self.k_bias), - i=torch.tensor(self.i_bias), - f=torch.tensor(self.f_bias), - round_mode=self.round_mode, + k=self.k_bias, + i=self.i_bias, + f=self.f_bias, overflow=self.overflow_mode_parameters, - is_data=False, + round_mode=self.round_mode, is_heterogeneous=self.use_hgq, - place="bias", + is_data=False, granularity=self.bias_quant_granularity, + place="bias", + shape=None if self._bias is None else self._bias.shape, ) if self.use_hgq: self.input_quantizer.quantizer.build(input_shape) @@ -978,7 +864,7 @@ def check_is_built(self, input_shape): def apply_final_compression(self): self._weight.data = self.weight self._bias.data = self.bias - self.final_compression_done = True + self.final_compression_done.fill_(True) def get_input_quantization_bits(self): return self.input_quantizer.get_quantization_bits() @@ -989,32 +875,32 @@ def get_weight_quantization_bits(self): def get_bias_quantization_bits(self): return self.bias_quantizer.get_quantization_bits() - def is_fitcompress_pretraining(self): + def _is_fitcompress_pretraining(self): return self.is_pretraining and self.use_fitcompress @property def weight(self): - if self.enable_quantization and not self.final_compression_done and not self.is_fitcompress_pretraining(): + if self.enable_quantization and not self.final_compression_done and not self._is_fitcompress_pretraining(): return self.weight_quantizer(self._weight) return self._weight @property def bias(self): - if self.enable_quantization and not self.final_compression_done and not self.is_fitcompress_pretraining(): + if self.enable_quantization and not self.final_compression_done and not self._is_fitcompress_pretraining(): return self.bias_quantizer(self._bias) return self._bias def ebops(self): bw_inp = self.input_quantizer.get_total_bits(self.input_shape) - bw_ker = ops.reshape(self.weight_quantizer.get_total_bits(self.running_mean.shape), self._shape) - bw_bias = ops.reshape(self.bias_quantizer.get_total_bits(self.running_mean.shape), self._shape) - size = ops.cast(ops.prod(list(self.input_shape)), self._weight.dtype) - ebops = ops.sum(bw_inp * bw_ker) + ops.mean(bw_bias) * size + bw_ker = torch.reshape(self.weight_quantizer.get_total_bits(self.running_mean.shape), self._shape) + bw_bias = torch.reshape(self.bias_quantizer.get_total_bits(self.running_mean.shape), self._shape) + size = float(math.prod(self.input_shape)) + ebops = torch.sum(bw_inp * bw_ker) + torch.mean(bw_bias) * size return ebops def hgq_loss(self): if self.is_pretraining or not self.use_hgq: - return ops.convert_to_tensor(0.0) + return torch.tensor(0.0) loss = self.hgq_beta * self.ebops() loss += self.weight_quantizer.hgq_loss() loss += self.bias_quantizer.hgq_loss() @@ -1026,17 +912,16 @@ def post_pre_train_function(self): self.is_pretraining = False def forward(self, input: torch.Tensor) -> torch.Tensor: - self.check_is_built(input.shape) + self._check_is_built(input.shape) if self.quantize_input and self.enable_quantization: - if not self.is_fitcompress_pretraining(): + if not self._is_fitcompress_pretraining(): input = self.input_quantizer(input) - else: - if self.post_fitcompress_calibration: - self.saved_inputs.append(input) + elif self.post_fitcompress_calibration: + self.saved_inputs.append(input) return super().forward(input) -class PQBatchNorm1d(nn.BatchNorm1d): +class PQBatchNorm2d(PQBatchNormBase, nn.BatchNorm2d): def __init__( self, config, @@ -1056,158 +941,48 @@ def __init__( bias_quant_granularity=None, ): super().__init__(num_features, eps, momentum, affine, track_running_stats, device=device, dtype=dtype) - if in_quant_bits is not None: - self.k_input, self.i_input, self.f_input = in_quant_bits - else: - self.k_input = config.quantization_parameters.default_data_keep_negatives - self.i_input = config.quantization_parameters.default_data_integer_bits - self.f_input = config.quantization_parameters.default_data_fractional_bits - - if weight_quant_bits is not None: - self.k_weight, self.i_weight, self.f_weight = weight_quant_bits - else: - self.k_weight = config.quantization_parameters.default_weight_keep_negatives - self.i_weight = config.quantization_parameters.default_weight_integer_bits - self.f_weight = config.quantization_parameters.default_weight_fractional_bits - if bias_quant_bits is not None: - self.k_bias, self.i_bias, self.f_bias = bias_quant_bits - else: - self.k_bias = config.quantization_parameters.default_weight_keep_negatives - self.i_bias = config.quantization_parameters.default_weight_integer_bits - self.f_bias = config.quantization_parameters.default_weight_fractional_bits - self.overflow_mode_parameters = config.quantization_parameters.overflow_mode_parameters - self.overflow_mode_data = config.quantization_parameters.overflow_mode_data - self.round_mode = config.quantization_parameters.round_mode - self.use_hgq = config.quantization_parameters.use_high_granularity_quantization - self.hgq_gamma = config.quantization_parameters.hgq_gamma - self.hgq_beta = config.quantization_parameters.hgq_beta - self.enable_quantization = config.quantization_parameters.enable_quantization - self.use_fitcompress = config.fitcompress_parameters.enable_fitcompress - self.config = config - self.quantize_input = quantize_input - granularity = config.quantization_parameters.granularity - self.in_quant_granularity = in_quant_granularity if in_quant_granularity is not None else granularity - self.weight_quant_granularity = weight_quant_granularity if weight_quant_granularity is not None else granularity - self.bias_quant_granularity = bias_quant_granularity if bias_quant_granularity is not None else granularity - self._weight = nn.Parameter(self.weight.clone()).to(self.weight.device) - self.register_parameter("_weight", self._weight) - if self.bias is not None: - self._bias = nn.Parameter(self.bias.clone()).to(self.bias.device) - self.register_parameter("_bias", self._bias) - else: - self.register_parameter("_bias", None) - self.register_parameter("_weight", self._weight) - self.built = False - self.final_compression_done = False - self.is_pretraining = True - self.post_fitcompress_calibration = False - self.saved_inputs = [] - - def check_is_built(self, input_shape): - if self.built: - return - self.built = True - self.input_quantizer = Quantizer( - k=torch.tensor(self.k_input), - i=torch.tensor(self.i_input), - f=torch.tensor(self.f_input), - overflow=self.overflow_mode_data, - round_mode=self.round_mode, - is_heterogeneous=self.use_hgq, - is_data=True, - hgq_gamma=self.hgq_gamma, - place="datalane", - dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, - granularity=self.in_quant_granularity, - ) - self.weight_quantizer = Quantizer( - k=torch.tensor(self.k_weight), - i=torch.tensor(self.i_weight), - f=torch.tensor(self.f_weight), - round_mode=self.round_mode, - overflow=self.overflow_mode_parameters, - is_data=False, - is_heterogeneous=self.use_hgq, - place="weight", - granularity=self.weight_quant_granularity, - ) - self.bias_quantizer = Quantizer( - k=torch.tensor(self.k_bias), - i=torch.tensor(self.i_bias), - f=torch.tensor(self.f_bias), - round_mode=self.round_mode, - overflow=self.overflow_mode_parameters, - is_data=False, - is_heterogeneous=self.use_hgq, - place="bias", - granularity=self.bias_quant_granularity, + self._init_quantization( + config, + quantize_input, + in_quant_bits, + weight_quant_bits, + bias_quant_bits, + in_quant_granularity, + weight_quant_granularity, + bias_quant_granularity, ) - if self.use_hgq: - self.input_quantizer.quantizer.build(input_shape) - shape = [1] * len(input_shape) - shape[1] = input_shape[1] - self._shape = tuple(shape) - self.input_shape = (1,) + input_shape[1:] - - def apply_final_compression(self): - self._weight.data = self.weight - self._bias.data = self.bias - self.final_compression_done = True - - def get_input_quantization_bits(self): - return self.input_quantizer.get_quantization_bits() - - def get_weight_quantization_bits(self): - return self.weight_quantizer.get_quantization_bits() - - def get_bias_quantization_bits(self): - return self.bias_quantizer.get_quantization_bits() - - def is_fitcompress_pretraining(self): - return self.is_pretraining and self.use_fitcompress - @property - def weight(self): - if self.enable_quantization and not self.final_compression_done and not self.is_fitcompress_pretraining(): - return self.weight_quantizer(self._weight) - return self._weight - - @property - def bias(self): - if self.enable_quantization and not self.final_compression_done and not self.is_fitcompress_pretraining(): - return self.bias_quantizer(self._bias) - return self._bias - - def ebops(self): - bw_inp = self.input_quantizer.get_total_bits(self.input_shape) - bw_ker = ops.reshape(self.weight_quantizer.get_total_bits(self.running_mean.shape), self._shape) - bw_bias = ops.reshape(self.bias_quantizer.get_total_bits(self.running_mean.shape), self._shape) - size = ops.cast(ops.prod(list(self.input_shape)), self._weight.dtype) - ebops = ops.sum(bw_inp * bw_ker) + ops.mean(bw_bias) * size - return ebops - - def hgq_loss(self): - if self.is_pretraining or not self.use_hgq: - return ops.convert_to_tensor(0.0) - loss = self.hgq_beta * self.ebops() - loss += self.weight_quantizer.hgq_loss() - loss += self.bias_quantizer.hgq_loss() - if self.quantize_input: - loss += self.input_quantizer.hgq_loss() - return loss - def post_pre_train_function(self): - self.is_pretraining = False - - def forward(self, input: torch.Tensor) -> torch.Tensor: - self.check_is_built(input.shape) - if self.quantize_input and self.enable_quantization: - if not self.is_fitcompress_pretraining(): - input = self.input_quantizer(input) - else: - if self.post_fitcompress_calibration: - self.saved_inputs.append(input) - return super().forward(input) +class PQBatchNorm1d(PQBatchNormBase, nn.BatchNorm1d): + def __init__( + self, + config, + num_features: int, + eps: float = 1e-5, + momentum: typing.Optional[float] = 0.1, + affine: bool = True, + track_running_stats: bool = True, + device=None, + dtype=None, + quantize_input=True, + in_quant_bits: Tuple[T, T, T] = None, + weight_quant_bits: Tuple[T, T, T] = None, + bias_quant_bits: Tuple[T, T, T] = None, + in_quant_granularity=None, + weight_quant_granularity=None, + bias_quant_granularity=None, + ): + super().__init__(num_features, eps, momentum, affine, track_running_stats, device=device, dtype=dtype) + self._init_quantization( + config, + quantize_input, + in_quant_bits, + weight_quant_bits, + bias_quant_bits, + in_quant_granularity, + weight_quant_granularity, + bias_quant_granularity, + ) class PQLayerNorm(nn.LayerNorm): @@ -1236,32 +1011,10 @@ def __init__( except TypeError: # Older torch versions don't accept the bias kwarg super().__init__(normalized_shape, eps, elementwise_affine, device=device, dtype=dtype) - if in_quant_bits is not None: - self.k_input, self.i_input, self.f_input = in_quant_bits - else: - self.k_input = config.quantization_parameters.default_data_keep_negatives - self.i_input = config.quantization_parameters.default_data_integer_bits - self.f_input = config.quantization_parameters.default_data_fractional_bits - - if out_quant_bits is not None: - self.k_output, self.i_output, self.f_output = out_quant_bits - else: - self.k_output = config.quantization_parameters.default_data_keep_negatives - self.i_output = config.quantization_parameters.default_data_integer_bits - self.f_output = config.quantization_parameters.default_data_fractional_bits - - if weight_quant_bits is not None: - self.k_weight, self.i_weight, self.f_weight = weight_quant_bits - else: - self.k_weight = config.quantization_parameters.default_weight_keep_negatives - self.i_weight = config.quantization_parameters.default_weight_integer_bits - self.f_weight = config.quantization_parameters.default_weight_fractional_bits - if bias_quant_bits is not None: - self.k_bias, self.i_bias, self.f_bias = bias_quant_bits - else: - self.k_bias = config.quantization_parameters.default_weight_keep_negatives - self.i_bias = config.quantization_parameters.default_weight_integer_bits - self.f_bias = config.quantization_parameters.default_weight_fractional_bits + self.k_input, self.i_input, self.f_input = _resolve_data_quant_bits(in_quant_bits, config) + self.k_output, self.i_output, self.f_output = _resolve_data_quant_bits(out_quant_bits, config) + self.k_weight, self.i_weight, self.f_weight = _resolve_weight_quant_bits(weight_quant_bits, config) + self.k_bias, self.i_bias, self.f_bias = _resolve_weight_quant_bits(bias_quant_bits, config) self.overflow_mode_parameters = config.quantization_parameters.overflow_mode_parameters self.overflow_mode_data = config.quantization_parameters.overflow_mode_data self.round_mode = config.quantization_parameters.round_mode @@ -1289,62 +1042,64 @@ def __init__( else: self.register_parameter("_bias", None) self.built = False - self.final_compression_done = False + self.register_buffer("final_compression_done", torch.tensor(False)) self.is_pretraining = True self.post_fitcompress_calibration = False self.saved_inputs = [] - def check_is_built(self, input_shape): + def _check_is_built(self, input_shape): if self.built: return self.built = True self.input_quantizer = Quantizer( - k=torch.tensor(self.k_input), - i=torch.tensor(self.i_input), - f=torch.tensor(self.f_input), + k=self.k_input, + i=self.i_input, + f=self.f_input, overflow=self.overflow_mode_data, round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=True, + granularity=self.in_quant_granularity, hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, - granularity=self.in_quant_granularity, ) self.output_quantizer = Quantizer( - k=torch.tensor(self.k_output), - i=torch.tensor(self.i_output), - f=torch.tensor(self.f_output), + k=self.k_output, + i=self.i_output, + f=self.f_output, overflow=self.overflow_mode_data, round_mode=self.round_mode, is_heterogeneous=self.use_hgq, is_data=True, + granularity=self.out_quant_granularity, hgq_gamma=self.hgq_gamma, place="datalane", dynamic_data=self.config.quantization_parameters.dynamic_data_quantization, - granularity=self.out_quant_granularity, ) self.weight_quantizer = Quantizer( - k=torch.tensor(self.k_weight), - i=torch.tensor(self.i_weight), - f=torch.tensor(self.f_weight), - round_mode=self.round_mode, + k=self.k_weight, + i=self.i_weight, + f=self.f_weight, overflow=self.overflow_mode_parameters, - is_data=False, + round_mode=self.round_mode, is_heterogeneous=self.use_hgq, - place="weight", + is_data=False, granularity=self.weight_quant_granularity, + place="weight", + shape=self._weight.shape, ) self.bias_quantizer = Quantizer( - k=torch.tensor(self.k_bias), - i=torch.tensor(self.i_bias), - f=torch.tensor(self.f_bias), - round_mode=self.round_mode, + k=self.k_bias, + i=self.i_bias, + f=self.f_bias, overflow=self.overflow_mode_parameters, - is_data=False, + round_mode=self.round_mode, is_heterogeneous=self.use_hgq, - place="bias", + is_data=False, granularity=self.bias_quant_granularity, + place="bias", + shape=None if self._bias is None else self._bias.shape, ) if self.use_hgq: self.input_quantizer.quantizer.build(input_shape) @@ -1356,7 +1111,7 @@ def apply_final_compression(self): self._weight.data = self.weight if self._bias is not None: self._bias.data = self.bias - self.final_compression_done = True + self.final_compression_done.fill_(True) def get_input_quantization_bits(self): return self.input_quantizer.get_quantization_bits() @@ -1370,14 +1125,14 @@ def get_weight_quantization_bits(self): def get_bias_quantization_bits(self): return self.bias_quantizer.get_quantization_bits() - def is_fitcompress_pretraining(self): + def _is_fitcompress_pretraining(self): return self.is_pretraining and self.use_fitcompress @property def weight(self): if self._weight is None: return None - if self.enable_quantization and not self.final_compression_done and not self.is_fitcompress_pretraining(): + if self.enable_quantization and not self.final_compression_done and not self._is_fitcompress_pretraining(): return self.weight_quantizer(self._weight) return self._weight @@ -1385,7 +1140,7 @@ def weight(self): def bias(self): if self._bias is None: return None - if self.enable_quantization and not self.final_compression_done and not self.is_fitcompress_pretraining(): + if self.enable_quantization and not self.final_compression_done and not self._is_fitcompress_pretraining(): return self.bias_quantizer(self._bias) return self._bias @@ -1394,7 +1149,7 @@ def ebops(self): def hgq_loss(self): if self.is_pretraining or not self.use_hgq: - return ops.convert_to_tensor(0.0) + return torch.tensor(0.0) loss = self.hgq_beta * self.ebops() if self._weight is not None: loss += self.weight_quantizer.hgq_loss() @@ -1410,15 +1165,14 @@ def post_pre_train_function(self): self.is_pretraining = False def forward(self, input: torch.Tensor) -> torch.Tensor: - self.check_is_built(input.shape) + self._check_is_built(input.shape) if self.quantize_input and self.enable_quantization: - if not self.is_fitcompress_pretraining(): + if not self._is_fitcompress_pretraining(): input = self.input_quantizer(input) - else: - if self.post_fitcompress_calibration: - self.saved_inputs.append(input) + elif self.post_fitcompress_calibration: + self.saved_inputs.append(input) out = F.layer_norm(input, self.normalized_shape, self.weight, self.bias, self.eps) - if self.quantize_output and self.enable_quantization and not self.is_fitcompress_pretraining(): + if self.quantize_output and self.enable_quantization and not self._is_fitcompress_pretraining(): out = self.output_quantizer(out) return out @@ -1542,8 +1296,8 @@ def _head_bits(self, proj, seq_len): bw = proj.output_quantizer.get_total_bits((1, seq_len, self.embed_dim)) return bw.reshape(1, seq_len, self.num_heads, self.head_dim).transpose(1, 2) - def _attention_ebops(self): - """EBOPs of the q @ k^T and attn @ v einsums (mirrors HGQ's QMultiHeadAttention._compute_ebops).""" + def attention_ebops(self): + """EBOPs of the q @ k^T and attn @ v einsums (mirrors HGQ's QMultiHeadAttention).""" attn_shape = self.softmax.input_shape # (1, H, T, S), stored when the softmax was built query_len, key_len = attn_shape[2], attn_shape[3] bw_q = self._head_bits(self.q_proj, query_len) @@ -1557,12 +1311,12 @@ def _attention_ebops(self): def ebops(self): # Only the attention einsum costs are this module's own: the projections, # softmax and lookup tables are counted by get_ebops's recursive modules() walk. - return self._attention_ebops() + return self.attention_ebops() def hgq_loss(self): if self.is_pretraining or not self.use_hgq: return torch.tensor(0.0) - return self.hgq_beta * self._attention_ebops() + return self.hgq_beta * self.attention_ebops() def forward( self, @@ -1630,160 +1384,55 @@ def extra_repr(self) -> str: ) -def add_layer_specific_quantization_to_model(name, layer, config): - if isinstance(layer, PQWeightBiasBase): - if name in config.quantization_parameters.layer_specific: - layer_config = config.quantization_parameters.layer_specific[name] - if "weight" in layer_config: - if "keep_negatives" in layer_config["weight"]: - layer.k_weight = torch.tensor(layer_config["weight"]["keep_negatives"]) - if "integer_bits" in layer_config["weight"]: - layer.i_weight = torch.tensor(layer_config["weight"]["integer_bits"]) - if "fractional_bits" in layer_config["weight"]: - layer.f_weight = torch.tensor(layer_config["weight"]["fractional_bits"]) - if "bias" in layer_config: - if "keep_negatives" in layer_config["bias"]: - layer.k_bias = torch.tensor(layer_config["bias"]["keep_negatives"]) - if "integer_bits" in layer_config["bias"]: - layer.i_bias = torch.tensor(layer_config["bias"]["integer_bits"]) - if "fractional_bits" in layer_config["bias"]: - layer.f_bias = torch.tensor(layer_config["bias"]["fractional_bits"]) - if "input" in layer_config: - if "keep_negatives" in layer_config["input"]: - input_keep_negatives = torch.tensor(layer_config["input"]["keep_negatives"]) - layer.k_input = input_keep_negatives - if "integer_bits" in layer_config["input"]: - input_int_bits = torch.tensor(layer_config["input"]["integer_bits"]) - layer.i_input = input_int_bits - if "fractional_bits" in layer_config["input"]: - input_fractional_bits = torch.tensor(layer_config["input"]["fractional_bits"]) - layer.f_input = input_fractional_bits - if "quantize" in layer_config["input"]: - quantize = layer_config["input"]["quantize"] - layer.quantize_input = quantize - if "output" in layer_config: - if "keep_negatives" in layer_config["input"]: - output_keep_negatives = torch.tensor(layer_config["output"]["keep_negatives"]) - layer.k_output = output_keep_negatives - if "integer_bits" in layer_config["output"]: - output_int_bits = torch.tensor(layer_config["output"]["integer_bits"]) - layer.i_output = input_int_bits - if "fractional_bits" in layer_config["output"]: - input_fractional_bits = torch.tensor(layer_config["output"]["fractional_bits"]) - layer.f_output = input_fractional_bits - if "quantize" in layer_config["output"]: - quantize = layer_config["output"]["quantize"] - layer.quantize_output = quantize - - elif layer.__class__ in [PQBatchNorm2d, PQBatchNorm1d]: - if name in config.quantization_parameters.layer_specific: - layer_config = config.quantization_parameters.layer_specific[name] - if "weight" in layer_config: - i = torch.tensor(layer_config["weight"]["integer_bits"]) - f = torch.tensor(layer_config["weight"]["fractional_bits"]) - layer.i_weight = i - layer.f_weight = f - if "bias" in layer_config: - i = torch.tensor(layer_config["bias"]["integer_bits"]) - f = torch.tensor(layer_config["bias"]["fractional_bits"]) - layer.i_bias = i - layer.f_bias = f - if "input" in layer_config: - if "integer_bits" in layer_config["input"]: - input_int_bits = torch.tensor(layer_config["input"]["integer_bits"]) - layer.i_input = input_int_bits - if "fractional_bits" in layer_config["input"]: - input_fractional_bits = torch.tensor(layer_config["input"]["fractional_bits"]) - layer.f_input = input_fractional_bits - if "quantize" in layer_config["input"]: - quantize = layer_config["input"]["quantize"] - layer.quantize_input = quantize - elif layer.__class__ == PQLayerNorm: - if name in config.quantization_parameters.layer_specific: - layer_config = config.quantization_parameters.layer_specific[name] - if "weight" in layer_config: - if "keep_negatives" in layer_config["weight"]: - layer.k_weight = torch.tensor(layer_config["weight"]["keep_negatives"]) - if "integer_bits" in layer_config["weight"]: - layer.i_weight = torch.tensor(layer_config["weight"]["integer_bits"]) - if "fractional_bits" in layer_config["weight"]: - layer.f_weight = torch.tensor(layer_config["weight"]["fractional_bits"]) - if "bias" in layer_config: - if "keep_negatives" in layer_config["bias"]: - layer.k_bias = torch.tensor(layer_config["bias"]["keep_negatives"]) - if "integer_bits" in layer_config["bias"]: - layer.i_bias = torch.tensor(layer_config["bias"]["integer_bits"]) - if "fractional_bits" in layer_config["bias"]: - layer.f_bias = torch.tensor(layer_config["bias"]["fractional_bits"]) - if "input" in layer_config: - if "keep_negatives" in layer_config["input"]: - layer.k_input = torch.tensor(layer_config["input"]["keep_negatives"]) - if "integer_bits" in layer_config["input"]: - layer.i_input = torch.tensor(layer_config["input"]["integer_bits"]) - if "fractional_bits" in layer_config["input"]: - layer.f_input = torch.tensor(layer_config["input"]["fractional_bits"]) - if "quantize" in layer_config["input"]: - layer.quantize_input = layer_config["input"]["quantize"] - if "output" in layer_config: - if "keep_negatives" in layer_config["output"]: - layer.k_output = torch.tensor(layer_config["output"]["keep_negatives"]) - if "integer_bits" in layer_config["output"]: - layer.i_output = torch.tensor(layer_config["output"]["integer_bits"]) - if "fractional_bits" in layer_config["output"]: - layer.f_output = torch.tensor(layer_config["output"]["fractional_bits"]) - if "quantize" in layer_config["output"]: - layer.quantize_output = layer_config["output"]["quantize"] - elif layer.__class__ in [PQAvgPool1d, PQAvgPool2d]: - if name in config.quantization_parameters.layer_specific: - layer_config = config.quantization_parameters.layer_specific[name] - if "input" in layer_config: - if "integer_bits" in layer_config["input"]: - input_int_bits = torch.tensor(layer_config["input"]["integer_bits"]) - layer.i_input = input_int_bits - if "fractional_bits" in layer_config["input"]: - input_fractional_bits = torch.tensor(layer_config["input"]["fractional_bits"]) - layer.f_input = input_fractional_bits - if "quantize" in layer_config["input"]: - quantize = layer_config["input"]["quantize"] - layer.quantize_input = quantize - if "output" in layer_config: - if "integer_bits" in layer_config["output"]: - output_int_bits = torch.tensor(layer_config["output"]["integer_bits"]) - layer.i_output = output_int_bits - if "fractional_bits" in layer_config["output"]: - output_fractional_bits = torch.tensor(layer_config["output"]["fractional_bits"]) - layer.f_output = output_fractional_bits - if "quantize" in layer_config["output"]: - quantize = layer_config["output"]["quantize"] - layer.quantize_output = quantize - - elif layer.__class__ == PQActivation: - if name in config.quantization_parameters.layer_specific: - layer_config = config.quantization_parameters.layer_specific[name] - if "input" in layer_config: - if "integer_bits" in layer_config["input"]: - input_int_bits = torch.tensor(layer_config["input"]["integer_bits"]) - layer.i_input = input_int_bits - if "fractional_bits" in layer_config["input"]: - input_fractional_bits = torch.tensor(layer_config["input"]["fractional_bits"]) - layer.f_input = input_fractional_bits - if "quantize" in layer_config["input"]: - quantize = layer_config["input"]["quantize"] - layer.quantize_input = quantize - if "output" in layer_config: - if "integer_bits" in layer_config["output"]: - output_int_bits = torch.tensor(layer_config["output"]["integer_bits"]) - layer.i_output = output_int_bits - if "fractional_bits" in layer_config["output"]: - output_fractional_bits = torch.tensor(layer_config["output"]["fractional_bits"]) - layer.f_output = output_fractional_bits - if "quantize" in layer_config["output"]: - quantize = layer_config["output"]["quantize"] - layer.quantize_output = quantize +# Prunable leaf layers, larger layers like MHA not included as they consist of these layers +LAYERS_WITH_PRUNING_LAYER = (PQConv2d, PQConv1d, PQDense) +PQ_MODULES = ( + PQConv2d, + PQConv1d, + PQDense, + PQActivation, + PQBatchNorm2d, + PQBatchNorm1d, + PQLayerNorm, + PQAvgPoolBase, + PQSoftmax, + PQMultiheadAttention, + Quantizer, +) + + +def _apply_quant_bits(layer, section_config, suffix): + """Copy keep_negatives/integer/fractional bits from one config section onto layer.{k,i,f}_{suffix}.""" + if "keep_negatives" in section_config: + setattr(layer, f"k_{suffix}", torch.tensor(section_config["keep_negatives"])) + if "integer_bits" in section_config: + setattr(layer, f"i_{suffix}", torch.tensor(section_config["integer_bits"])) + if "fractional_bits" in section_config: + setattr(layer, f"f_{suffix}", torch.tensor(section_config["fractional_bits"])) + + +def _add_layer_specific_quantization_to_model(name, layer, config): + layer_config = config.quantization_parameters.layer_specific.get(name) + if layer_config is None: + return layer + if isinstance(layer, (PQWeightBiasBase, PQLayerNorm)): + sections = ("weight", "bias", "input", "output") + elif isinstance(layer, (PQBatchNormBase)): + sections = ("weight", "bias", "input") + elif isinstance(layer, (PQAvgPoolBase, PQActivation)): + sections = ("input", "output") + else: + return layer + for section in sections: + if section not in layer_config: + continue + _apply_quant_bits(layer, layer_config[section], section) + if section in ("input", "output") and "quantize" in layer_config[section]: + setattr(layer, f"quantize_{section}", layer_config[section]["quantize"]) return layer -def add_quantized_activations_to_model_layer(module, config, prefix=""): +def _add_quantized_activations_to_model_layer(module, config, prefix=""): if not config.quantization_parameters.enable_quantization: return module quantize_input = config.quantization_parameters.quantize_input @@ -1805,7 +1454,7 @@ def add_quantized_activations_to_model_layer(module, config, prefix=""): quantize_input=quantize_input, quantize_output=quantize_output, ) - relu = add_layer_specific_quantization_to_model(full_name, relu, config) + relu = _add_layer_specific_quantization_to_model(full_name, relu, config) setattr(module, name, relu) elif layer.__class__ in [nn.Tanh]: type_of_tanh = "tanh" if config.quantization_parameters.use_real_tanh else "hard_tanh" @@ -1817,7 +1466,7 @@ def add_quantized_activations_to_model_layer(module, config, prefix=""): quantize_input=quantize_input, quantize_output=quantize_output, ) - tanh = add_layer_specific_quantization_to_model(full_name, tanh, config) + tanh = _add_layer_specific_quantization_to_model(full_name, tanh, config) setattr(module, name, tanh) elif layer.__class__ == nn.AvgPool1d: new_layer = PQAvgPool1d( @@ -1830,7 +1479,7 @@ def add_quantized_activations_to_model_layer(module, config, prefix=""): quantize_input, quantize_output, ) - new_layer = add_layer_specific_quantization_to_model(full_name, new_layer, config) + new_layer = _add_layer_specific_quantization_to_model(full_name, new_layer, config) setattr(module, name, new_layer) elif layer.__class__ == nn.AvgPool2d: new_layer = PQAvgPool2d( @@ -1844,10 +1493,11 @@ def add_quantized_activations_to_model_layer(module, config, prefix=""): quantize_input, quantize_output, ) - new_layer = add_layer_specific_quantization_to_model(full_name, new_layer, config) + new_layer = _add_layer_specific_quantization_to_model(full_name, new_layer, config) setattr(module, name, new_layer) - elif layer.__class__ == nn.BatchNorm2d: - new_layer = PQBatchNorm2d( + elif layer.__class__ in (nn.BatchNorm1d, nn.BatchNorm2d): + pq_batchnorm = PQBatchNorm1d if layer.__class__ is nn.BatchNorm1d else PQBatchNorm2d + new_layer = pq_batchnorm( config, num_features=layer.num_features, eps=layer.eps, @@ -1856,19 +1506,7 @@ def add_quantized_activations_to_model_layer(module, config, prefix=""): track_running_stats=layer.track_running_stats, quantize_input=quantize_input, ) - new_layer = add_layer_specific_quantization_to_model(full_name, new_layer, config) - setattr(module, name, new_layer) - elif layer.__class__ == nn.BatchNorm1d: - new_layer = PQBatchNorm1d( - config, - num_features=layer.num_features, - eps=layer.eps, - momentum=layer.momentum, - affine=layer.affine, - track_running_stats=layer.track_running_stats, - quantize_input=quantize_input, - ) - new_layer = add_layer_specific_quantization_to_model(full_name, new_layer, config) + new_layer = _add_layer_specific_quantization_to_model(full_name, new_layer, config) setattr(module, name, new_layer) elif layer.__class__ == nn.LayerNorm: ln_kwargs = dict( @@ -1884,54 +1522,29 @@ def add_quantized_activations_to_model_layer(module, config, prefix=""): new_layer._weight.data.copy_(layer.weight.data) if layer.bias is not None and new_layer._bias is not None: new_layer._bias.data.copy_(layer.bias.data) - new_layer = add_layer_specific_quantization_to_model(full_name, new_layer, config) + new_layer = _add_layer_specific_quantization_to_model(full_name, new_layer, config) setattr(module, name, new_layer) else: - layer = add_quantized_activations_to_model_layer(layer, config, full_name) + layer = _add_quantized_activations_to_model_layer(layer, config, full_name) return module -def add_quantized_activations_to_model_functional(module, config): - # Currently not in use. TODO: Fix this - if config.quantization_parameters.use_high_granularity_quantization: - return module - # Replaces functional activation calls with quantized versions - traced_model = symbolic_trace(module) - for node in traced_model.graph.nodes: - if node.op in ["call_method", "call_function"] and (node.target == "tanh" or "function relu" in str(node.target)): - with traced_model.graph.inserting_after(node): - if node.name in config.quantization_parameters.layer_specific: - bits = config.quantization_parameters.layer_specific[node.name]["bits"] - else: - bits = ( - config.quantization_parameters.default_integer_bits - + config.quantization_parameters.default_fractional_bits - + 1 - ) # 1 sign bit - kwargs = {"bits": bits} - if node.target == "tanh": - kwargs["use_real_tanh"] = config.quantization_parameters.use_real_tanh - kwargs["use_symmetric"] = config.quantization_parameters.use_symmetric_quantization - # new_node = traced_model.graph.call_function(quantized_tanh, node.args, kwargs) - else: - kwargs = {"integer_bits": config.quantization_parameters.default_integer_bits, "bits": bits} - # new_node = traced_model.graph.call_function(quantized_relu, node.args, kwargs) - # node.replace_all_uses_with(new_node) - traced_model.graph.erase_node(node) - - traced_model.graph.lint() - traced_model.recompile() - return traced_model - - -def disable_pruning_from_layers(name, layer, config): - enable_pruning = name not in config.pruning_parameters.disable_pruning_for_layers - if layer.__class__ in [PQDense, PQConv2d, PQConv1d] and not enable_pruning: - layer.enable_pruning = enable_pruning +def _disable_pruning_from_layers(name, layer, config): + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER) and name in config.pruning_parameters.disable_pruning_for_layers: + layer.enable_pruning = False return layer -def add_pruning_to_model(module, config, prefix=""): +def _replace_layer_with_pq_layer(module, name, full_name, layer, sparse_layer, config): + sparse_layer._weight.data = layer.weight.data + if layer.bias is not None: + sparse_layer._bias.data = layer.bias.data + sparse_layer = _add_layer_specific_quantization_to_model(full_name, sparse_layer, config) + sparse_layer = _disable_pruning_from_layers(full_name, sparse_layer, config) + setattr(module, name, sparse_layer) + + +def _add_pruning_to_model(module, config, prefix=""): quantize_input = config.quantization_parameters.quantize_input quantize_output = config.quantization_parameters.quantize_output for name, layer in module.named_children(): @@ -1940,15 +1553,10 @@ def add_pruning_to_model(module, config, prefix=""): sparse_layer = PQDense( config, layer.in_features, layer.out_features, layer.bias is not None, quantize_input, quantize_output ) - sparse_layer._weight.data = layer.weight.data - if layer.bias is not None: - sparse_layer._bias.data = layer.bias.data - - sparse_layer = add_layer_specific_quantization_to_model(full_name, sparse_layer, config) - sparse_layer = disable_pruning_from_layers(full_name, sparse_layer, config) - setattr(module, name, sparse_layer) - elif layer.__class__ is nn.Conv2d: - sparse_layer = PQConv2d( + _replace_layer_with_pq_layer(module, name, full_name, layer, sparse_layer, config) + elif layer.__class__ in (nn.Conv1d, nn.Conv2d): + pq_conv = PQConv1d if layer.__class__ is nn.Conv1d else PQConv2d + sparse_layer = pq_conv( config, layer.in_channels, layer.out_channels, @@ -1964,37 +1572,39 @@ def add_pruning_to_model(module, config, prefix=""): quantize_input, quantize_output, ) - sparse_layer._weight.data = layer.weight.data - if layer.bias is not None: - sparse_layer._bias.data = layer.bias.data - sparse_layer = add_layer_specific_quantization_to_model(full_name, sparse_layer, config) - sparse_layer = disable_pruning_from_layers(full_name, sparse_layer, config) - setattr(module, name, sparse_layer) - elif layer.__class__ is nn.Conv1d: - sparse_layer = PQConv1d( + _replace_layer_with_pq_layer(module, name, full_name, layer, sparse_layer, config) + elif layer.__class__ is nn.MultiheadAttention: + if layer.bias_k is not None or layer.add_zero_attn: + raise ValueError(f"add_bias_kv/add_zero_attn are not supported by PQMultiheadAttention ({full_name})") + sparse_layer = PQMultiheadAttention( config, - layer.in_channels, - layer.out_channels, - layer.kernel_size, - layer.stride, - layer.padding, - layer.dilation, - layer.groups, - layer.bias is not None, - layer.padding_mode, - layer.weight.device, - layer.weight.dtype, - quantize_input, - quantize_output, + embed_dim=layer.embed_dim, + num_heads=layer.num_heads, + dropout=layer.dropout, + bias=layer.in_proj_bias is not None, + kdim=layer.kdim, + vdim=layer.vdim, + batch_first=layer.batch_first, + quantize_input=quantize_input, + quantize_output=quantize_output, ) - sparse_layer._weight.data = layer.weight.data - if layer.bias is not None: - sparse_layer._bias.data = layer.bias.data - sparse_layer = add_layer_specific_quantization_to_model(full_name, sparse_layer, config) - sparse_layer = disable_pruning_from_layers(full_name, sparse_layer, config) + if layer._qkv_same_embed_dim: + q_w, k_w, v_w = layer.in_proj_weight.chunk(3) + else: + q_w, k_w, v_w = layer.q_proj_weight, layer.k_proj_weight, layer.v_proj_weight + if layer.in_proj_bias is not None: + biases = (*layer.in_proj_bias.chunk(3), layer.out_proj.bias) + else: + biases = (None, None, None, None) + projs = (sparse_layer.q_proj, sparse_layer.k_proj, sparse_layer.v_proj, sparse_layer.out_proj) + for proj, weight, bias in zip(projs, (q_w, k_w, v_w, layer.out_proj.weight), biases): + proj._weight.data = weight.data.clone() + if bias is not None: + proj._bias.data = bias.data.clone() + sparse_layer = _add_layer_specific_quantization_to_model(full_name, sparse_layer, config) setattr(module, name, sparse_layer) else: - add_pruning_to_model(layer, config, full_name) + _add_pruning_to_model(layer, config, full_name) return module @@ -2015,127 +1625,92 @@ def call_post_round_functions(model, rewind, rounds, r): post_round_functions(model) +def _update_pruning_mask(layer): + if layer.enable_pruning and hasattr(layer.pruning_layer, "update_mask"): + layer.pruning_layer.update_mask(layer._weight) + + def post_epoch_functions(model, epoch, total_epochs, **kwargs): for layer in model.modules(): - if isinstance(layer, (PQConv2d, PQConv1d, PQDense)): + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): layer.pruning_layer.post_epoch_function(epoch, total_epochs, **kwargs) + _update_pruning_mask(layer) elif isinstance(layer, Quantizer): layer.post_epoch_function() def pre_epoch_functions(model, epoch, total_epochs): for layer in model.modules(): - if isinstance(layer, (PQConv2d, PQConv1d, PQDense)): + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): layer.pruning_layer.pre_epoch_function(epoch, total_epochs) def post_round_functions(model): for layer in model.modules(): - if isinstance(layer, (PQConv2d, PQConv1d, PQDense)): + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): layer.pruning_layer.post_round_function() def save_weights_functions(model): for layer in model.modules(): - if isinstance(layer, (PQConv2d, PQConv1d, PQDense)): - layer.save_weights() + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): + layer._save_weights() def rewind_weights_functions(model): for layer in model.modules(): - if isinstance(layer, (PQConv2d, PQConv1d, PQDense)): - layer.rewind_weights() + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): + layer._rewind_weights() def pre_finetune_functions(model): for layer in model.modules(): - if isinstance(layer, (PQConv2d, PQConv1d, PQDense)): + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): layer.pruning_layer.pre_finetune_function() def post_pretrain_functions(model, config, train_loader=None, loss_function=None, input_shape=None): + for layer in model.modules(): + if isinstance(layer, PQ_MODULES): + # For FITCompress this must happen before the compression path search, + # so quantization is already enabled during it. + layer.post_pre_train_function() if config.fitcompress_parameters.enable_fitcompress: from pquant.core.torch.fit_compress import call_fitcompress # noqa: 811 - for layer in model.modules(): - if isinstance( - layer, - ( - PQConv2d, - PQConv1d, - PQDense, - PQActivation, - PQBatchNorm2d, - PQBatchNorm1d, - PQLayerNorm, - PQAvgPoolBase, - PQSoftmax, - PQMultiheadAttention, - Quantizer, - ), - ): - # Trigger it here to enable quantization before FITCompress - layer.post_pre_train_function() config, pruning_mask_importance_scores = call_fitcompress( config, model, train_loader, loss_function, input_shape=input_shape ) idx = 0 for layer in model.modules(): - if isinstance(layer, (PQConv2d, PQConv1d, PQDense)): - # layer.post_pre_train_function() - # set_data_quantization_bits(model) + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): with torch.no_grad(): layer.pruning_layer.mask.data = pruning_mask_importance_scores[idx] layer.pruning_layer.pre_finetune_function() # So mask is not updated during training anymore idx += 1 return - else: - for layer in model.modules(): - if isinstance( - layer, - ( - PQConv2d, - PQConv1d, - PQDense, - PQActivation, - PQBatchNorm2d, - PQBatchNorm1d, - PQLayerNorm, - PQAvgPoolBase, - PQSoftmax, - PQMultiheadAttention, - Quantizer, - ), - ): - layer.post_pre_train_function() if config.pruning_parameters.pruning_method == "pdp" or ( config.pruning_parameters.pruning_method == "wanda" and config.pruning_parameters.calculate_pruning_budget ): - # pass - pdp_setup(model, config) + _pdp_setup(model, config) -def pdp_setup(model, config): +def _pdp_setup(model, config): """ Calculates a global sparsity threshold. Initializes target sparsity for each layer, which depends on how large percentage of weights in the layer is smaller than the global threshold """ - global_weights = None - for layer in model.modules(): - if isinstance(layer, (PQConv2d, PQConv1d, PQDense)): - if global_weights is None: - global_weights = layer._weight.flatten() - else: - global_weights = torch.concat((global_weights, layer._weight.flatten())) - + global_weights = torch.concat( + [layer._weight.flatten() for layer in model.modules() if isinstance(layer, LAYERS_WITH_PRUNING_LAYER)] + ) abs_global_weights = torch.abs(global_weights) global_weight_topk, _ = torch.topk(abs_global_weights, abs_global_weights.numel()) threshold = global_weight_topk[int((1 - config.pruning_parameters.sparsity) * global_weight_topk.numel())] global_weights_below_threshold = torch.where(abs_global_weights < threshold, 1, 0) idx = 0 for layer in model.modules(): - if isinstance(layer, (PQConv2d, PQConv1d, PQDense)): + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): weight_size = layer._weight.numel() w = torch.sum(global_weights_below_threshold[idx : idx + weight_size]) layer.pruning_layer.init_r = w / weight_size @@ -2148,11 +1723,10 @@ def get_layer_keep_ratio(model): total_w = 0 remaining_weights = 0 for layer in model.modules(): - if isinstance(layer, (PQConv2d, PQConv1d, PQDense)): + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): weight = layer.weight - total_w += ops.size(weight) - rem = ops.count_nonzero(weight) - remaining_weights += rem + total_w += weight.numel() + remaining_weights += torch.count_nonzero(weight) elif layer.__class__ in (nn.Conv2d, nn.Conv1d, nn.Linear): total_w += layer.weight.numel() remaining_weights += torch.count_nonzero(layer.weight) @@ -2161,19 +1735,17 @@ def get_layer_keep_ratio(model): return 0.0 -def is_training_stage(layer): - return False if layer.pruning_layer._is_finetuning or layer.pruning_layer._is_pretraining else True +def _is_training_stage(layer): + return not (layer.pruning_layer._is_finetuning or layer.pruning_layer._is_pretraining) def get_model_losses(model, losses): for layer in model.modules(): - loss = 0.0 - if isinstance(layer, (PQConv2d, PQConv1d, PQDense)): - if layer.enable_pruning and is_training_stage(layer) and not layer.use_fitcompress: - loss += layer.pruning_layer.calculate_additional_loss() + if isinstance(layer, LAYERS_WITH_PRUNING_LAYER): + if layer.enable_pruning and _is_training_stage(layer) and not layer.use_fitcompress: + losses += layer.pruning_layer.calculate_additional_loss() if layer.use_hgq: - loss += layer.hgq_loss() - losses += loss + losses += layer.hgq_loss() elif isinstance( layer, ( @@ -2192,131 +1764,83 @@ def get_model_losses(model, losses): return losses -def create_default_layer_quantization_pruning_config(model, config): - # subconfig = {"layer_specific": {}, "disable_pruning_for_layers": []} +def _create_default_layer_quantization_pruning_config(model, config): + quant_params = config.quantization_parameters + + def data_section(quantize, integer_bits, fractional_bits): + return { + "quantize": quantize, + "keep_negatives": quant_params.default_data_keep_negatives, + "integer_bits": integer_bits, + "fractional_bits": fractional_bits, + } + + def param_section(integer_bits, fractional_bits): + return { + "keep_negatives": quant_params.default_weight_keep_negatives, + "integer_bits": integer_bits, + "fractional_bits": fractional_bits, + } + for name, layer in model.named_modules(): if layer.__class__ in [nn.Linear, nn.Conv1d, nn.Conv2d]: - if layer.bias is None: - config.quantization_parameters.layer_specific[name] = { - "input": { - "keep_negatives": config.quantization_parameters.default_data_keep_negatives, - "integer_bits": 0, - "fractional_bits": 7, - "quantize": config.quantization_parameters.quantize_input, - }, - "weight": { - "keep_negatives": config.quantization_parameters.default_weight_keep_negatives, - "integer_bits": 0, - "fractional_bits": 7, - }, - "output": { - "keep_negatives": config.quantization_parameters.default_data_keep_negatives, - "integer_bits": 0, - "fractional_bits": 7, - "quantize": config.quantization_parameters.quantize_output, - }, - } - else: - config.quantization_parameters.layer_specific[name] = { - "input": { - "keep_negatives": config.quantization_parameters.default_data_keep_negatives, - "integer_bits": 0, - "fractional_bits": 7, - "quantize": config.quantization_parameters.quantize_input, - }, - "weight": { - "keep_negatives": config.quantization_parameters.default_weight_keep_negatives, - "integer_bits": 0, - "fractional_bits": 7, - }, - "bias": { - "keep_negatives": config.quantization_parameters.default_weight_keep_negatives, - "integer_bits": 0, - "fractional_bits": 7, - }, - "output": { - "keep_negatives": config.quantization_parameters.default_data_keep_negatives, - "integer_bits": 0, - "fractional_bits": 7, - "quantize": config.quantization_parameters.quantize_output, - }, - } + layer_config = { + "input": data_section(quant_params.quantize_input, 0, 7), + "weight": param_section(0, 7), + "output": data_section(quant_params.quantize_output, 0, 7), + } + if layer.bias is not None: + layer_config["bias"] = param_section(0, 7) + quant_params.layer_specific[name] = layer_config config.pruning_parameters.disable_pruning_for_layers.append(name) elif layer.__class__ in [nn.Tanh, nn.ReLU, nn.AvgPool1d, nn.AvgPool2d, nn.AvgPool3d]: - config.quantization_parameters.layer_specific[name] = { - "input": { - "quantize": config.quantization_parameters.quantize_input, - "keep_negatives": config.quantization_parameters.default_data_keep_negatives, - "integer_bits": 0.0, - "fractional_bits": 7.0, - }, - "output": { - "quantize": config.quantization_parameters.quantize_output, - "keep_negatives": config.quantization_parameters.default_data_keep_negatives, - "integer_bits": 0.0, - "fractional_bits": 7.0, - }, + quant_params.layer_specific[name] = { + "input": data_section(quant_params.quantize_input, 0.0, 7.0), + "output": data_section(quant_params.quantize_output, 0.0, 7.0), } elif layer.__class__ in [nn.BatchNorm2d]: - config.quantization_parameters.layer_specific[name] = { - "input": { - "quantize": config.quantization_parameters.quantize_input, - "keep_negatives": config.quantization_parameters.default_data_keep_negatives, - "integer_bits": 0.0, - "fractional_bits": 7.0, - }, - "weight": { - "keep_negatives": config.quantization_parameters.default_weight_keep_negatives, - "integer_bits": 0, - "fractional_bits": 7.0, - }, - "bias": { - "keep_negatives": config.quantization_parameters.default_weight_keep_negatives, - "integer_bits": 0, - "fractional_bits": 7.0, - }, + quant_params.layer_specific[name] = { + "input": data_section(quant_params.quantize_input, 0.0, 7.0), + "weight": param_section(0, 7.0), + "bias": param_section(0, 7.0), } return config def populate_config_with_all_layers(model, config): - return create_default_layer_quantization_pruning_config(model, config) + return _create_default_layer_quantization_pruning_config(model, config) -def remove_compression_layers(module, config): +def _remove_compression_layers(module, config): for name, layer in module.named_children(): if isinstance(layer, PQDense): - out_features = layer.out_features - in_features = layer.in_features - bias = True if layer.bias is not None else False - setattr(module, name, nn.Linear(in_features=in_features, out_features=out_features, bias=bias)) - getattr(module, name).weight.data.copy_(layer.weight) - if getattr(module, name).bias is not None: - getattr(module, name).bias.data.copy_(layer.bias) + new_layer = nn.Linear( + in_features=layer.in_features, out_features=layer.out_features, bias=layer.bias is not None + ) + new_layer.weight.data.copy_(layer.weight) + if new_layer.bias is not None: + new_layer.bias.data.copy_(layer.bias) + setattr(module, name, new_layer) elif isinstance(layer, (PQConv1d, PQConv2d)): - bias_values = layer.bias if layer.bias is not None else None - bias = True if bias_values is not None else False + bias_values = layer.bias conv = nn.Conv2d if isinstance(layer, PQConv2d) else nn.Conv1d - setattr( - module, - name, - conv( - layer.in_channels, - layer.out_channels, - layer.kernel_size, - layer.stride, - layer.padding, - layer.dilation, - layer.groups, - bias, - layer.padding_mode, - ), + new_layer = conv( + layer.in_channels, + layer.out_channels, + layer.kernel_size, + layer.stride, + layer.padding, + layer.dilation, + layer.groups, + bias_values is not None, + layer.padding_mode, ) - getattr(module, name).weight.data.copy_(layer.weight) - if getattr(module, name).bias is not None: - getattr(module, name).bias.data.copy_(bias_values.data) + new_layer.weight.data.copy_(layer.weight) + if new_layer.bias is not None: + new_layer.bias.data.copy_(bias_values.data) + setattr(module, name, new_layer) else: - remove_compression_layers(layer, config) + _remove_compression_layers(layer, config) return module @@ -2329,25 +1853,25 @@ def post_training_prune(model, config, calibration_data): model = add_compression_layers(model, config, inputs.shape) post_pretrain_functions(model, config) model(inputs) - return remove_compression_layers(model, config) + return _remove_compression_layers(model, config) def get_ebops(model, **kwargs): ebops = 0 for m in model.modules(): - if isinstance(m, (PQWeightBiasBase)): + if isinstance(m, PQWeightBiasBase): ebops += m.ebops(include_mask=m.enable_pruning) elif isinstance( - m, (PQAvgPoolBase, PQBatchNorm1d, PQBatchNorm2d, PQLayerNorm, PQActivation, PQSoftmax, PQMultiheadAttention) + m, + ( + PQAvgPoolBase, + PQBatchNorm1d, + PQBatchNorm2d, + PQLayerNorm, + PQActivation, + PQSoftmax, + PQMultiheadAttention, + ), ): ebops += m.ebops() return ebops - - -def load_torch_hgq_model(model, path_to_checkpoint): - model.load_state_dict(torch.load(path_to_checkpoint), strict=False) - for m in model.modules(): - if isinstance(m, Quantizer) and m.quantizer.built: - # Populate HGQ quantizer bit values from PQuantML quantizer - m.reload_from_local() - return model diff --git a/src/pquant/core/torch/pruning_methods/fitcompress.py b/src/pquant/core/torch/pruning_methods/fitcompress.py index 7a31d69..2c119fe 100644 --- a/src/pquant/core/torch/pruning_methods/fitcompress.py +++ b/src/pquant/core/torch/pruning_methods/fitcompress.py @@ -10,8 +10,8 @@ def __init__(self, config, *args, **kwargs): config = PQConfig.load_from_config(config) self.config = config - self.is_pretraining = True - self.is_finetuning = False + self._is_pretraining = True + self._is_finetuning = False self.built = False def build(self, input_shape): @@ -33,13 +33,13 @@ def calculate_additional_loss(self): return 0.0 def pre_finetune_function(self): - self.is_finetuning = True + self._is_finetuning = True def post_round_function(self): pass def post_pre_train_function(self): - self.is_pretraining = False + self._is_pretraining = False def post_epoch_function(self, epoch, total_epochs, **kwargs): pass diff --git a/src/pquant/core/torch/pruning_methods/pdp.py b/src/pquant/core/torch/pruning_methods/pdp.py index 5204988..3fa097a 100644 --- a/src/pquant/core/torch/pruning_methods/pdp.py +++ b/src/pquant/core/torch/pruning_methods/pdp.py @@ -124,9 +124,13 @@ def _mask_structured_channel(self, weight): def forward(self, weight): if self._is_pretraining or self._is_finetuning: return self.mask.to(weight.dtype) * weight - new_mask = self._compute_mask(weight) - self.mask.data = new_mask - return self.mask * weight + return self._compute_mask(weight) * weight + + def update_mask(self, weight): + """Update stored mask from current weights. Called once per epoch from post_epoch_functions.""" + if not self._is_pretraining and not self._is_finetuning: + with torch.no_grad(): + self.mask.copy_(self._compute_mask(weight)) def get_hard_mask(self, weight=None): return (self.mask >= 0.5).to(self.mask.dtype) diff --git a/src/pquant/core/torch/quantizer.py b/src/pquant/core/torch/quantizer.py index e2ff630..b664547 100644 --- a/src/pquant/core/torch/quantizer.py +++ b/src/pquant/core/torch/quantizer.py @@ -1,8 +1,7 @@ -from enum import Enum - import torch import torch.nn as nn +from pquant.core.constants import QuantizationGranularity from pquant.core.torch.fixed_point_quantizer import get_fixed_quantizer from pquant.core.torch.hgq_quantizer import HGQQuantizer @@ -17,27 +16,28 @@ def __init__( round_mode, is_heterogeneous, is_data=False, - granularity='per_tensor', + granularity=QuantizationGranularity.PER_TENSOR, hgq_gamma=0, place="datalane", dynamic_data=True, + shape=None, ): super().__init__() - self.k = torch.nn.Parameter(torch.tensor(float(k)), requires_grad=False) + self.overflow = overflow - self.b_init = k + i + f self.round_mode = round_mode self.use_hgq = is_heterogeneous self.is_data = is_data self.dynamic_data = dynamic_data - self.i_init = i - self.f_init = f - self.i = torch.nn.Parameter(torch.tensor(i), requires_grad=False) - self.f = torch.nn.Parameter(torch.tensor(f), requires_grad=False) - self.b = torch.nn.Parameter(torch.tensor(i + k + f), requires_grad=False) - self.granularity = granularity.value if isinstance(granularity, Enum) else granularity + self.granularity = QuantizationGranularity(granularity).value + if not self.use_hgq: + param_shape = () if is_data else self.compute_weight_param_shape(shape) + self.k = torch.nn.Parameter(torch.full(param_shape, float(k)), requires_grad=False) + self.i = torch.nn.Parameter(torch.full(param_shape, float(i)), requires_grad=False) + self.f = torch.nn.Parameter(torch.full(param_shape, float(f)), requires_grad=False) + self.b = torch.nn.Parameter(torch.full(param_shape, float(i + k + f)), requires_grad=False) self.quantizer = create_quantizer( - self.k, + k, i, f, self.overflow, @@ -49,9 +49,7 @@ def __init__( ) self.is_pretraining = True self.hgq_gamma = hgq_gamma - self.final_compression_done = nn.Parameter(torch.tensor(False), requires_grad=False) - if self.granularity == 'per_tensor': - self.initialize_quantization_parameters(self.i_init, self.f_init) + self.register_buffer("final_compression_done", torch.tensor(False)) def get_quantization_bits(self): if self.use_hgq: @@ -69,18 +67,17 @@ def get_total_bits(self, shape): def set_quantization_bits(self, i, f): if self.use_hgq: self.quantizer.set_bits(i, f) - self.i.data = torch.tensor(i) - self.f.data = torch.tensor(f) + else: + self.i.data = torch.as_tensor(i, dtype=self.i.dtype, device=self.i.device).broadcast_to(self.i.shape).clone() + self.f.data = torch.as_tensor(f, dtype=self.f.dtype, device=self.f.device).broadcast_to(self.f.shape).clone() def post_pre_train_function(self): self.is_pretraining = False def calculate_bits_from_abs(self, abs_x): m = torch.ceil(torch.log2(abs_x + 1e-6)) - int_bits = torch.clamp(m, min=0) - b = self.b if hasattr(self, "b") else self.k + self.i_init + self.f_init - int_bits = torch.clamp(m, max=b - self.k.to(m.device)) - frac_bits = torch.clamp(b - int_bits - self.k, min=0) + int_bits = torch.clamp(m, min=0).clamp(max=self.b - self.k.to(m.device)) + frac_bits = torch.clamp(self.b - int_bits - self.k, min=0) return int_bits, frac_bits def compute_data_dynamic_bits(self, x): @@ -90,18 +87,26 @@ def compute_data_dynamic_bits(self, x): abs_x = torch.amax(torch.abs(x)) return self.calculate_bits_from_abs(abs_x) + def compute_weight_param_shape(self, shape): + if shape is None or self.granularity == QuantizationGranularity.PER_TENSOR or len(shape) == 1: + return () + elif self.granularity == QuantizationGranularity.PER_CHANNEL: + return (shape[0],) + (1,) * (len(shape) - 1) # Channels first + else: + return shape + def compute_weight_dynamic_bits(self, x): - if self.granularity == "per_tensor" or x.ndim == 1: + if self.granularity == QuantizationGranularity.PER_TENSOR or x.ndim == 1 or not self.training: _, i, f = self.get_quantization_bits() return i, f - if self.granularity == "per_channel": + if self.granularity == QuantizationGranularity.PER_CHANNEL: if x.ndim == 2: abs_x = torch.amax(torch.abs(x), dim=1, keepdim=True) elif x.ndim == 3: abs_x = torch.amax(torch.abs(x), dim=(1, 2), keepdim=True) elif x.ndim == 4: abs_x = torch.amax(torch.abs(x), dim=(1, 2, 3), keepdim=True) - elif self.granularity == "per_weight": + elif self.granularity == QuantizationGranularity.PER_WEIGHT: abs_x = torch.abs(x) else: raise ValueError("The selected granularity is not supported.") @@ -114,16 +119,13 @@ def compute_dynamic_bits(self, x): def forward(self, x): if self.use_hgq: - x = self.quantizer(x, training=self.training) - _, i, f = self.get_quantization_bits() - self.initialize_quantization_parameters(i, f) - return x + return self.quantizer(x, training=self.training) + elif self.final_compression_done: + return self.quantizer(x, k=self.k, i=self.i, f=self.f, training=False) else: i, f = self.compute_dynamic_bits(x) - self.initialize_quantization_parameters(i, f) self.i.data = i self.f.data = f - _, i, f = self.get_quantization_bits() x = self.quantizer(x, k=self.k, i=i, f=f, training=self.training) return x @@ -144,27 +146,18 @@ def apply_final_compression(self): self.quantizer._f.data.clamp_(self.quantizer.f_min, self.quantizer.f_max) if self.quantizer.overflow_mode != "WRAP": self.quantizer._i.data.clamp_(self.quantizer.i_min, self.quantizer.i_max) + self.final_compression_done.fill_(True) + return _, i, f = self.get_quantization_bits() self.i.data = i self.f.data = f self.b.data = i + f - self.final_compression_done.data = torch.tensor(True) - - def initialize_quantization_parameters(self, i, f): - if hasattr(self, "f"): - return - # Lazy initialization - self.i = torch.nn.Parameter(torch.tensor(i), requires_grad=False) - self.f = torch.nn.Parameter(torch.tensor(f), requires_grad=False) - self.b = torch.nn.Parameter(torch.tensor(self.k.detach().clone() + i + f), requires_grad=False) - - def reload_from_local(self): - if not self.use_hgq: - return - self.quantizer.set_bits(self.i, self.f) + self.final_compression_done.fill_(True) -def create_quantizer(k, i, f, overflow, round_mode, is_heterogeneous, is_data, granularity="per_weight", gamma=1e-8): +def create_quantizer( + k, i, f, overflow, round_mode, is_heterogeneous, is_data, granularity=QuantizationGranularity.PER_WEIGHT, gamma=1e-8 +): if is_heterogeneous: return HGQQuantizer( k0=k, diff --git a/src/pquant/core/torch/tracing.py b/src/pquant/core/torch/tracing.py index 0bbc30e..a3dfdf5 100644 --- a/src/pquant/core/torch/tracing.py +++ b/src/pquant/core/torch/tracing.py @@ -3,6 +3,7 @@ import torch import torch.nn.functional as F +from pquant.core.constants import QuantizationGranularity from pquant.core.torch.activations import PQActivation from pquant.core.torch.layers import ( PQAvgPoolBase, @@ -263,7 +264,7 @@ def _make_quantizer(): round_mode=qp.round_mode, is_heterogeneous=False, is_data=True, - granularity="per_tensor", + granularity=QuantizationGranularity.PER_TENSOR, hgq_gamma=qp.hgq_gamma, ) diff --git a/src/pquant/core/torch/utils.py b/src/pquant/core/torch/utils.py index 8f0f0d7..6f6f466 100644 --- a/src/pquant/core/torch/utils.py +++ b/src/pquant/core/torch/utils.py @@ -2,6 +2,7 @@ from pquant.core.torch.pruning_methods.autosparse import AutoSparse from pquant.core.torch.pruning_methods.cs import ContinuousSparsification from pquant.core.torch.pruning_methods.dst import DST +from pquant.core.torch.pruning_methods.fitcompress import FITCompress from pquant.core.torch.pruning_methods.mdmm import MDMM from pquant.core.torch.pruning_methods.pdp import PDP from pquant.core.torch.pruning_methods.wanda import Wanda @@ -15,6 +16,8 @@ def get_pruning_layer(config, layer_type): return AutoSparse(config, layer_type) elif pruning_method == "cs": return ContinuousSparsification(config, layer_type) + elif pruning_method == "fitcompress": + return FITCompress(config) elif pruning_method == "pdp": return PDP(config, layer_type) elif pruning_method == "activation_pruning": diff --git a/src/pquant/data_models/quantization_model.py b/src/pquant/data_models/quantization_model.py index db8e2db..bcd3976 100644 --- a/src/pquant/data_models/quantization_model.py +++ b/src/pquant/data_models/quantization_model.py @@ -1,12 +1,6 @@ -from enum import Enum - from pydantic import BaseModel, Field - -class QuantizationGranularity(str, Enum): - PER_TENSOR = "per_tensor" - PER_CHANNEL = "per_channel" - PER_WEIGHT = "per_weight" +from pquant.core.constants import QuantizationGranularity class BaseQuantizationModel(BaseModel): diff --git a/tests/conftest.py b/tests/conftest.py index f581100..8e377b0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,10 +8,7 @@ @pytest.fixture(scope="function", autouse=True) def set_image_data_format(): - if "DATA_FORMAT" in os.environ: - keras.backend.set_image_data_format(os.environ["DATA_FORMAT"]) - else: - keras.backend.set_image_data_format("channels_first") + keras.backend.set_image_data_format(os.environ.get("DATA_FORMAT", "channels_last")) @pytest.fixture(scope='session', autouse=True, params=[42]) diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 66119ec..34fedbc 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -6,8 +6,9 @@ pytest test_pdp.py KERAS_BACKEND="torch" pytest test_pdp.py pytest test_wanda.py KERAS_BACKEND="torch" pytest test_wanda.py +pytest test_torch_pruning_layers.py +pytest test_quantizer_parity.py pytest test_keras_compression_layers.py -DATA_FORMAT=channels_last pytest test_keras_compression_layers.py KERAS_BACKEND="torch" pytest test_torch_compression_layers.py KERAS_BACKEND=torch pytest test_torch_onnx_converter.py pytest test_keras_onnx_converter.py diff --git a/tests/test_keras_compression_layers.py b/tests/test_keras_compression_layers.py index 53d272b..1d3be67 100644 --- a/tests/test_keras_compression_layers.py +++ b/tests/test_keras_compression_layers.py @@ -35,6 +35,7 @@ PQConv2d, PQDense, PQDepthwiseConv2d, + PQMultiheadAttention, PQSeparableConv2d, add_compression_layers, apply_final_compression, @@ -2136,3 +2137,149 @@ def test_model_fit(config_fn): model.compile(optimizer="adam", loss="mse", jit_compile=False) callback = PQuantCallback(config, log_ebops=False, log_keep_ratio=False) model.fit(dummy_x, dummy_y, epochs=callback.total_epochs, callbacks=[callback], verbose=0) + + +def build_single_layer_model(layer_type, config): + channels_first = keras.backend.image_data_format() == "channels_first" + conv2d_shape = (IN_FEATURES, 8, 8) if channels_first else (8, 8, IN_FEATURES) + conv1d_shape = (IN_FEATURES, STEPS) if channels_first else (STEPS, IN_FEATURES) + bn_axis = 1 if channels_first else -1 + + if layer_type == "dense": + inputs = keras.Input(shape=(IN_FEATURES,)) + outputs = PQDense(config, units=OUT_FEATURES)(inputs) + elif layer_type == "conv1d": + inputs = keras.Input(shape=conv1d_shape) + outputs = PQConv1d(config, OUT_FEATURES, KERNEL_SIZE, padding="same")(inputs) + elif layer_type == "conv2d": + inputs = keras.Input(shape=conv2d_shape) + outputs = PQConv2d(config, OUT_FEATURES, KERNEL_SIZE, padding="same")(inputs) + elif layer_type == "depthwise_conv2d": + inputs = keras.Input(shape=conv2d_shape) + outputs = PQDepthwiseConv2d(config, KERNEL_SIZE, padding="same")(inputs) + elif layer_type == "separable_conv2d": + inputs = keras.Input(shape=conv2d_shape) + outputs = PQSeparableConv2d(config, OUT_FEATURES, KERNEL_SIZE, padding="same")(inputs) + elif layer_type == "batchnorm": + inputs = keras.Input(shape=conv2d_shape) + outputs = PQBatchNormalization(config, axis=bn_axis)(inputs) + elif layer_type == "avgpool1d": + inputs = keras.Input(shape=conv1d_shape) + outputs = PQAvgPool1d(config, pool_size=2, strides=2, padding="same")(inputs) + elif layer_type == "avgpool2d": + inputs = keras.Input(shape=conv2d_shape) + outputs = PQAvgPool2d(config, pool_size=2, strides=2, padding="same")(inputs) + elif layer_type in ("activation_relu", "activation_tanh", "activation_hard_tanh"): + activation = layer_type.replace("activation_", "") + inputs = keras.Input(shape=(IN_FEATURES,)) + outputs = PQActivation(config, activation=activation, quantize_input=True, quantize_output=True)(inputs) + elif layer_type == "mha": + inputs = keras.Input(shape=(STEPS, IN_FEATURES)) + outputs = PQMultiheadAttention(config, embed_dim=IN_FEATURES, num_heads=2)(inputs)[0] + else: + raise ValueError(f"unknown layer type {layer_type}") + + model = keras.Model(inputs, outputs) + dummy = np.zeros((1,) + tuple(inputs.shape[1:]), dtype=np.float32) + return model, dummy + + +def randomize_weights(model, seed): + rng = np.random.default_rng(seed) + for w in model.weights: + # Lifecycle flags must keep their 0/1 values. + if w.name.endswith(("is_pretraining", "is_finetuning")): + continue + w.assign(rng.standard_normal(w.shape).astype(w.dtype)) + + +def assert_weights_equal(model, reloaded, label): + assert len(model.weights) == len(reloaded.weights), f"[{label}] weight count mismatch" + for orig_w, loaded_w in zip(model.weights, reloaded.weights): + np.testing.assert_array_equal( + np.array(orig_w), + np.array(loaded_w), + err_msg=f"[{label}] Weight mismatch: {orig_w.name}", + ) + + +LAYER_TYPES = [ + "dense", + "conv1d", + "conv2d", + "depthwise_conv2d", + "separable_conv2d", + "batchnorm", + "avgpool1d", + "avgpool2d", + "activation_relu", + "activation_tanh", + "activation_hard_tanh", + "mha", +] + + +@pytest.mark.parametrize("use_hgq", [False, True], ids=["kif", "hgq"]) +@pytest.mark.parametrize("layer_type", LAYER_TYPES) +def test_layer_type_serialization(tmp_path, layer_type, use_hgq): + """Every PQ layer type must survive a full .keras save/load round-trip. + + Covers all quantizer variables (k/i/f/b, HGQ bitwidth variables) and pruning + state carried as model weights, at each lifecycle stage. + """ + # dst: no global pruning setup, so single-layer models without a prunable + # weight layer (batchnorm/pool/activation/mha) pass the lifecycle functions. + config = dst_config() + config.quantization_parameters.enable_quantization = True + config.quantization_parameters.use_high_granularity_quantization = use_hgq + + model, dummy = build_single_layer_model(layer_type, config) + model(dummy) + randomize_weights(model, seed=42) + + def roundtrip(label): + save_path = tmp_path / f"{label}.keras" + model.save(save_path) + reloaded = keras.models.load_model(save_path) + assert_weights_equal(model, reloaded, f"{layer_type}/{label}") + # The reloaded model must be usable, not just structurally equal. + np.testing.assert_allclose( + np.array(model(dummy)), + np.array(reloaded(dummy)), + rtol=0, + atol=0, + err_msg=f"[{layer_type}/{label}] forward pass mismatch after reload", + ) + + roundtrip("initial") + post_pretrain_functions(model, config) + roundtrip("post_pretrain") + pre_finetune_functions(model) + roundtrip("pre_finetune") + apply_final_compression(model) + roundtrip("final_compression") + + +@pytest.mark.parametrize("use_hgq", [False, True], ids=["kif", "hgq"]) +@pytest.mark.parametrize("layer_type", LAYER_TYPES) +def test_layer_type_checkpoint_save_load(tmp_path, layer_type, use_hgq): + """Every PQ layer type must survive a save_weights/load_weights round-trip.""" + # dst: no global pruning setup, so single-layer models without a prunable + # weight layer (batchnorm/pool/activation/mha) pass the lifecycle functions. + config = dst_config() + config.quantization_parameters.enable_quantization = True + config.quantization_parameters.use_high_granularity_quantization = use_hgq + + model, dummy = build_single_layer_model(layer_type, config) + model(dummy) + randomize_weights(model, seed=0) + original_weights = [np.array(w) for w in model.weights] + + path = str(tmp_path / "ckpt.weights.h5") + model.save_weights(path) + for w in model.weights: + w.assign(np.zeros(w.shape, dtype=w.dtype)) + model.load_weights(path) + + for orig, w in zip(original_weights, model.weights): + np.testing.assert_array_equal(orig, np.array(w), err_msg=f"[{layer_type}] Checkpoint weight mismatch: {w.name}") diff --git a/tests/test_quantizer_parity.py b/tests/test_quantizer_parity.py new file mode 100644 index 0000000..baf879f --- /dev/null +++ b/tests/test_quantizer_parity.py @@ -0,0 +1,357 @@ +""" +Parity tests: PyTorch `Quantizer` vs Keras `Quantizer` in non-HGQ (fixed-point) mode. + +Both implementations must produce matching forward outputs, matching gradients +w.r.t. the quantized tensor, and must freeze their dynamically-computed +bitwidths identically once training ends (``apply_final_compression``). + +``per_channel`` granularity needs special handling: keras stores weights +output-channel-last while torch stores them output-channel-first, so the raw +``Quantizer`` reduces over different axes in each backend. A real layer +reconciles this with a transpose before quantizing (keras ``_handle_transpose``); +here we do the same transpose manually (see ``to_channel_last``/``to_channel_first``) +so both backends reduce over the same channel groups before comparing. +""" + +import os + +os.environ.setdefault("KERAS_BACKEND", "tensorflow") + +import numpy as np # noqa: E402 +import pytest # noqa: E402 +import tensorflow as tf # noqa: E402 +import torch # noqa: E402 +from keras import ops # noqa: E402 + +from pquant.core.keras.quantizer import Quantizer as KQuantizer # noqa: E402 +from pquant.core.torch.quantizer import Quantizer as TQuantizer # noqa: E402 + +ABSOLUTE_TOLERANCE = 1e-5 +RELATIVE_TOLERANCE = 1e-4 + + +def to_numpy(x): + if isinstance(x, torch.Tensor): + return x.detach().cpu().numpy() + if isinstance(x, np.ndarray): + return x + return np.asarray(ops.convert_to_numpy(x)) + + +def to_channel_last(x): + """Move torch's channel-first axis (0) to keras's channel-last axis (-1).""" + return np.moveaxis(x, 0, -1) + + +def to_channel_first(x): + """Move keras's channel-last axis (-1) back to torch's channel-first axis (0).""" + return np.moveaxis(to_numpy(x), -1, 0) + + +def assert_close(a, b, atol=ABSOLUTE_TOLERANCE, rtol=RELATIVE_TOLERANCE, msg=""): + a_np = to_numpy(a) + b_np = to_numpy(b) + assert a_np.shape == b_np.shape, f"{msg}: shape mismatch: {a_np.shape} vs {b_np.shape}" + np.testing.assert_allclose(a_np, b_np, atol=atol, rtol=rtol, err_msg=msg) + + +def keras_tensor(arr): + return ops.convert_to_tensor(np.asarray(arr).astype(np.float32)) + + +def torch_tensor(arr, requires_grad=False): + return torch.as_tensor(np.asarray(arr).astype(np.float32)).requires_grad_(requires_grad) + + +def reset_seed(seed=0): + np.random.seed(seed) + torch.manual_seed(seed) + + +def keras_grad(fn, x, training=True): + """Gradient of scalar-reducing ``fn(x).sum()`` w.r.t. keras tensor ``x``.""" + xt = tf.convert_to_tensor(x) + with tf.GradientTape() as tape: + tape.watch(xt) + loss = ops.sum(fn(xt, training=training)) + return tape.gradient(loss, xt) + + +def make_quantizers(shape, k=1.0, i=2.0, f=2.0, overflow="SAT", round_mode="RND", is_data=False, granularity="per_tensor"): + k_layer = KQuantizer( + k=k, + i=i, + f=f, + overflow=overflow, + round_mode=round_mode, + is_heterogeneous=False, + is_data=is_data, + granularity=granularity, + ) + k_layer.build(shape) + + t_layer = TQuantizer( + k=k, + i=i, + f=f, + overflow=overflow, + round_mode=round_mode, + is_heterogeneous=False, + is_data=is_data, + granularity=granularity, + ) + return k_layer, t_layer + + +@pytest.mark.parametrize( + "shape,granularity,is_data", + [ + ((8, 4), "per_tensor", False), + ((8, 4), "per_weight", False), + ((4, 8, 3, 3), "per_weight", False), + ((16,), "per_tensor", True), + ], +) +def test_quantizer_matches_keras(shape, granularity, is_data): + reset_seed() + x_np = (np.random.randn(*shape) * 2.0).astype(np.float32) + + k_layer, t_layer = make_quantizers(shape, is_data=is_data, granularity=granularity) + + k_x = keras_tensor(x_np) + t_x = torch_tensor(x_np, requires_grad=True) + k_out = k_layer(k_x, training=True) + t_out = t_layer(t_x) + assert_close(k_out, t_out, msg=f"Quantizer forward ({granularity}, is_data={is_data})") + assert_close(k_layer.i, t_layer.i, msg=f"Quantizer i ({granularity}, is_data={is_data})") + assert_close(k_layer.f, t_layer.f, msg=f"Quantizer f ({granularity}, is_data={is_data})") + + k_grad = keras_grad(k_layer, x_np, training=True) + t_out.sum().backward() + assert_close(k_grad, t_x.grad, msg=f"Quantizer backward ({granularity}, is_data={is_data})") + + +@pytest.mark.parametrize("shape", [(8, 4), (4, 8, 3, 3)]) +def test_quantizer_per_channel_matches_keras(shape): + reset_seed() + x_np = (np.random.randn(*shape) * 2.0).astype(np.float32) + x_np_channel_last = to_channel_last(x_np) + + k_layer, t_layer = make_quantizers(x_np_channel_last.shape, is_data=False, granularity="per_channel") + + k_x = keras_tensor(x_np_channel_last) + t_x = torch_tensor(x_np, requires_grad=True) + k_out = k_layer(k_x, training=True) + t_out = t_layer(t_x) + assert_close(to_channel_first(k_out), t_out, msg=f"Quantizer per_channel forward {shape}") + assert_close(to_channel_first(k_layer.i), t_layer.i, msg=f"Quantizer per_channel i {shape}") + assert_close(to_channel_first(k_layer.f), t_layer.f, msg=f"Quantizer per_channel f {shape}") + + k_grad = keras_grad(k_layer, x_np_channel_last, training=True) + t_out.sum().backward() + assert_close(to_channel_first(k_grad), t_x.grad, msg=f"Quantizer per_channel backward {shape}") + + +@pytest.mark.parametrize("shape", [(8, 4), (4, 8, 3, 3)]) +def test_quantizer_per_channel_freezes_bits_after_final_compression(shape): + reset_seed() + train_np = (np.random.randn(*shape) * 2.0).astype(np.float32) + eval_np = (np.random.randn(*shape) * 500.0).astype(np.float32) + train_np_channel_last = to_channel_last(train_np) + eval_np_channel_last = to_channel_last(eval_np) + + k_layer, t_layer = make_quantizers(train_np_channel_last.shape, is_data=False, granularity="per_channel") + + k_layer(keras_tensor(train_np_channel_last), training=True) + t_layer.train() + t_layer(torch_tensor(train_np)) + + k_layer.apply_final_compression() + t_layer.apply_final_compression() + assert_close(to_channel_first(k_layer.i), t_layer.i, msg=f"per_channel i after apply_final_compression {shape}") + assert_close(to_channel_first(k_layer.f), t_layer.f, msg=f"per_channel f after apply_final_compression {shape}") + + frozen_k_i, frozen_k_f = to_numpy(k_layer.i).copy(), to_numpy(k_layer.f).copy() + + k_layer(keras_tensor(eval_np_channel_last), training=False) + t_layer.eval() + t_layer(torch_tensor(eval_np)) + + assert_close(k_layer.i, frozen_k_i, msg=f"keras per_channel i drifted after eval forward {shape}") + assert_close(k_layer.f, frozen_k_f, msg=f"keras per_channel f drifted after eval forward {shape}") + assert_close(t_layer.i, to_channel_first(frozen_k_i), msg=f"torch per_channel i drifted after eval forward {shape}") + assert_close(t_layer.f, to_channel_first(frozen_k_f), msg=f"torch per_channel f drifted after eval forward {shape}") + + +@pytest.mark.parametrize("granularity", ["per_tensor", "per_channel", "per_weight"]) +def test_is_data_ignores_granularity(granularity): + reset_seed() + shape = (8, 4) + x_np = (np.random.randn(*shape) * 2.0).astype(np.float32) + + k_layer, t_layer = make_quantizers(shape, is_data=True, granularity=granularity) + + k_out = k_layer(keras_tensor(x_np), training=True) + t_out = t_layer(torch_tensor(x_np)) + + assert to_numpy(k_layer.i).size == 1, f"keras is_data i should stay scalar for granularity={granularity}" + assert to_numpy(t_layer.i).size == 1, f"torch is_data i should stay scalar for granularity={granularity}" + assert_close(k_out, t_out, msg=f"Quantizer forward (is_data=True, granularity={granularity})") + + +@pytest.mark.parametrize("overflow", ["SAT", "SAT_SYM", "WRAP", "WRAP_SM"]) +def test_quantizer_matches_keras_overflow_modes(overflow): + reset_seed() + shape = (8, 4) + x_np = (np.random.randn(*shape) * 2.0).astype(np.float32) + + k_layer, t_layer = make_quantizers(shape, overflow=overflow, is_data=False, granularity="per_tensor") + + k_x = keras_tensor(x_np) + t_x = torch_tensor(x_np, requires_grad=True) + k_out = k_layer(k_x, training=True) + t_out = t_layer(t_x) + assert_close(k_out, t_out, msg=f"Quantizer forward (overflow={overflow})") + + k_grad = keras_grad(k_layer, x_np, training=True) + t_out.sum().backward() + assert_close(k_grad, t_x.grad, msg=f"Quantizer backward (overflow={overflow})") + + +def test_quantizer_matches_keras_wrap_eval_saturates(): + """WRAP overflow only saturates when training=False -- exercise that branch specifically.""" + reset_seed() + shape = (8, 4) + x_np = (np.random.randn(*shape) * 50.0).astype(np.float32) # large enough to trigger saturation + + k_layer, t_layer = make_quantizers(shape, overflow="WRAP", is_data=False, granularity="per_tensor") + + k_out = k_layer(keras_tensor(x_np), training=False) + t_layer.eval() + t_out = t_layer(torch_tensor(x_np)) + assert_close(k_out, t_out, msg="Quantizer WRAP eval-mode saturate branch") + + +@pytest.mark.parametrize( + "round_mode", + ["RND", "TRN", "RND_CONV", "TRN_ZERO", "RND_ZERO", "RND_MIN_INF", "RND_INF"], +) +def test_quantizer_matches_keras_round_modes(round_mode): + reset_seed() + shape = (8, 4) + x_np = (np.random.randn(*shape) * 2.0).astype(np.float32) + + k_layer, t_layer = make_quantizers(shape, round_mode=round_mode, is_data=False, granularity="per_tensor") + + k_x = keras_tensor(x_np) + t_x = torch_tensor(x_np, requires_grad=True) + k_out = k_layer(k_x, training=True) + t_out = t_layer(t_x) + assert_close(k_out, t_out, msg=f"Quantizer forward (round_mode={round_mode})") + + k_grad = keras_grad(k_layer, x_np, training=True) + t_out.sum().backward() + assert_close(k_grad, t_x.grad, msg=f"Quantizer backward (round_mode={round_mode})") + + +@pytest.mark.parametrize("k", [0.0, 1.0]) +def test_quantizer_matches_keras_k_values(k): + reset_seed() + shape = (8, 4) + x_np = (np.random.randn(*shape) * 2.0).astype(np.float32) + if k == 0.0: + x_np = np.abs(x_np) + + k_layer, t_layer = make_quantizers(shape, k=k, is_data=False, granularity="per_tensor") + + k_x = keras_tensor(x_np) + t_x = torch_tensor(x_np, requires_grad=True) + k_out = k_layer(k_x, training=True) + t_out = t_layer(t_x) + assert_close(k_out, t_out, msg=f"Quantizer forward (k={k})") + + k_grad = keras_grad(k_layer, x_np, training=True) + t_out.sum().backward() + assert_close(k_grad, t_x.grad, msg=f"Quantizer backward (k={k})") + + +def test_get_total_bits_matches_keras_non_hgq(): + reset_seed() + shape = (8, 4) + x_np = (np.random.randn(*shape) * 2.0).astype(np.float32) + + k_layer, t_layer = make_quantizers(shape, is_data=False, granularity="per_tensor") + k_layer(keras_tensor(x_np), training=True) + t_layer(torch_tensor(x_np)) + + assert_close(k_layer.get_total_bits(shape), t_layer.get_total_bits(shape), msg="get_total_bits (non-HGQ)") + + +def test_get_total_bits_matches_keras_hgq(): + reset_seed() + shape = (8, 4) + x_np = (np.random.randn(*shape) * 2.0).astype(np.float32) + + k_layer = KQuantizer( + k=1.0, + i=2.0, + f=2.0, + overflow="SAT", + round_mode="RND", + is_heterogeneous=True, + is_data=False, + granularity="per_tensor", + place="weight", + ) + k_layer.build(shape) + t_layer = TQuantizer( + k=1.0, + i=2.0, + f=2.0, + overflow="SAT", + round_mode="RND", + is_heterogeneous=True, + is_data=False, + granularity="per_tensor", + place="weight", + ) + + k_layer(keras_tensor(x_np), training=True) + t_layer(torch_tensor(x_np)) + + assert_close(k_layer.get_total_bits(shape), t_layer.get_total_bits(shape), msg="get_total_bits (HGQ)") + + +@pytest.mark.parametrize( + "shape,granularity", + [ + ((8, 4), "per_weight"), + ((4, 8, 3, 3), "per_weight"), + ], +) +def test_quantizer_freezes_bits_after_final_compression(shape, granularity): + reset_seed() + train_np = (np.random.randn(*shape) * 2.0).astype(np.float32) + eval_np = (np.random.randn(*shape) * 500.0).astype(np.float32) + + k_layer, t_layer = make_quantizers(shape, is_data=False, granularity=granularity) + + k_layer(keras_tensor(train_np), training=True) + t_layer.train() + t_layer(torch_tensor(train_np)) + + k_layer.apply_final_compression() + t_layer.apply_final_compression() + assert_close(k_layer.i, t_layer.i, msg=f"i after apply_final_compression ({granularity})") + assert_close(k_layer.f, t_layer.f, msg=f"f after apply_final_compression ({granularity})") + + frozen_k_i, frozen_k_f = to_numpy(k_layer.i).copy(), to_numpy(k_layer.f).copy() + + k_layer(keras_tensor(eval_np), training=False) + t_layer.eval() + t_layer(torch_tensor(eval_np)) + + assert_close(k_layer.i, frozen_k_i, msg=f"keras i drifted after eval forward ({granularity})") + assert_close(k_layer.f, frozen_k_f, msg=f"keras f drifted after eval forward ({granularity})") + assert_close(t_layer.i, frozen_k_i, msg=f"torch i drifted after eval forward ({granularity})") + assert_close(t_layer.f, frozen_k_f, msg=f"torch f drifted after eval forward ({granularity})") diff --git a/tests/test_torch_checkpoint.py b/tests/test_torch_checkpoint.py new file mode 100644 index 0000000..0c664d2 --- /dev/null +++ b/tests/test_torch_checkpoint.py @@ -0,0 +1,171 @@ +import os + +import numpy as np +import pytest +import torch +from torch import nn + +os.environ["KERAS_BACKEND"] = "torch" + +from pquant import dst_config # noqa: E402 +from pquant.activations import PQActivation # noqa: E402 +from pquant.layers import ( # noqa: E402 + PQAvgPool1d, + PQAvgPool2d, + PQBatchNorm1d, + PQBatchNorm2d, + PQConv1d, + PQConv2d, + PQDense, + PQLayerNorm, + PQMultiheadAttention, + apply_final_compression, + post_pretrain_functions, + pre_finetune_functions, +) + +BATCH_SIZE = 2 +OUT_FEATURES = 8 +IN_FEATURES = 4 +KERNEL_SIZE = 3 +STEPS = 6 + + +class SingleLayerModel(nn.Module): + + def __init__(self, layer, is_mha=False): + super().__init__() + self.layer = layer + self.is_mha = is_mha + + def forward(self, x): + if self.is_mha: + x, _ = self.layer(x, x, x) + else: + x = self.layer(x) + return x + + +def build_model_and_input(layer_type, config): + if layer_type == "dense": + layer = PQDense(config, IN_FEATURES, OUT_FEATURES) + x = torch.randn(BATCH_SIZE, IN_FEATURES) + elif layer_type == "conv1d": + layer = PQConv1d(config, IN_FEATURES, OUT_FEATURES, KERNEL_SIZE, padding=1) + x = torch.randn(BATCH_SIZE, IN_FEATURES, STEPS) + elif layer_type == "conv2d": + layer = PQConv2d(config, IN_FEATURES, OUT_FEATURES, KERNEL_SIZE, padding=1) + x = torch.randn(BATCH_SIZE, IN_FEATURES, STEPS, STEPS) + elif layer_type == "batchnorm1d": + layer = PQBatchNorm1d(config, IN_FEATURES) + x = torch.randn(BATCH_SIZE, IN_FEATURES, STEPS) + elif layer_type == "batchnorm2d": + layer = PQBatchNorm2d(config, IN_FEATURES) + x = torch.randn(BATCH_SIZE, IN_FEATURES, STEPS, STEPS) + elif layer_type == "layernorm": + layer = PQLayerNorm(config, IN_FEATURES) + x = torch.randn(BATCH_SIZE, STEPS, IN_FEATURES) + elif layer_type == "avgpool1d": + layer = PQAvgPool1d(config, kernel_size=2) + x = torch.randn(BATCH_SIZE, IN_FEATURES, STEPS) + elif layer_type == "avgpool2d": + layer = PQAvgPool2d(config, kernel_size=2) + x = torch.randn(BATCH_SIZE, IN_FEATURES, STEPS, STEPS) + elif layer_type.startswith("activation_"): + layer = PQActivation(config, activation=layer_type.replace("activation_", ""), quantize_output=True) + x = torch.randn(BATCH_SIZE, IN_FEATURES) + elif layer_type == "mha": + layer = PQMultiheadAttention(config, embed_dim=IN_FEATURES, num_heads=2) + x = torch.randn(STEPS, BATCH_SIZE, IN_FEATURES) + return SingleLayerModel(layer, is_mha=True), x + else: + raise ValueError(f"unknown layer kind {layer_type}") + + return SingleLayerModel(layer), x + + +STAGE_FLAGS = ("is_pretraining", "is_finetuning", "final_compression_done") + + +def randomize_state(model, seed): + gen = torch.Generator().manual_seed(seed) + with torch.no_grad(): + for name, t in model.state_dict().items(): + if any(k in name for k in STAGE_FLAGS): + continue + if not torch.is_floating_point(t): + continue + t.copy_(torch.rand(t.shape, generator=gen, device="cpu") + 0.5) + + +def advance_to_stage(model, config, stage): + if stage == "initial": + return + post_pretrain_functions(model, config) + if stage == "post_pretrain": + return + pre_finetune_functions(model) + if stage == "pre_finetune": + return + apply_final_compression(model) + assert stage == "final_compression" + + +def make_config(use_hgq): + config = dst_config() + config.quantization_parameters.enable_quantization = True + config.quantization_parameters.use_high_granularity_quantization = use_hgq + return config + + +LAYER_TYPES = [ + "dense", + "conv1d", + "conv2d", + "batchnorm1d", + "batchnorm2d", + "layernorm", + "avgpool1d", + "avgpool2d", + "activation_relu", + "activation_tanh", + "activation_hard_tanh", + "mha", +] +STAGES = ["initial", "post_pretrain", "pre_finetune", "final_compression"] + + +@pytest.mark.parametrize("use_hgq", [False, True], ids=["kif", "hgq"]) +@pytest.mark.parametrize("stage", STAGES) +@pytest.mark.parametrize("layer_type", LAYER_TYPES) +def test_state_dict_roundtrip(tmp_path, layer_type, stage, use_hgq): + torch.manual_seed(0) + config = make_config(use_hgq) + + model, x = build_model_and_input(layer_type, config) + model(x) # HGQ quantizers build lazily on first forward + advance_to_stage(model, config, stage) + randomize_state(model, seed=42) + + path = tmp_path / "ckpt.pt" + torch.save(model.state_dict(), path) + torch.manual_seed(1) + fresh_config = make_config(use_hgq) + fresh, _ = build_model_and_input(layer_type, fresh_config) + fresh(x) + advance_to_stage(fresh, fresh_config, stage) + missing, unexpected = fresh.load_state_dict(torch.load(path, weights_only=True), strict=True) + assert not missing and not unexpected + + saved = model.state_dict() + reloaded = fresh.state_dict() + assert saved.keys() == reloaded.keys() + for name in saved: + np.testing.assert_array_equal( + reloaded[name].cpu(), saved[name].cpu(), err_msg=f"state mismatch: {name}", strict=True + ) + + model.eval() + fresh.eval() + with torch.no_grad(): + np.testing.assert_array_equal(fresh(x).cpu(), model(x).cpu(), strict=True) diff --git a/tests/test_torch_pruning_layers.py b/tests/test_torch_pruning_layers.py index 495dd15..c7e7b59 100644 --- a/tests/test_torch_pruning_layers.py +++ b/tests/test_torch_pruning_layers.py @@ -21,85 +21,90 @@ from keras import ops # noqa: E402 from pquant.core.keras.pruning_methods.activation_pruning import ( # noqa: E402 - ActivationPruning as KActivationPruning, + ActivationPruning as KerasActivationPruning, ) from pquant.core.keras.pruning_methods.autosparse import ( # noqa: E402 - AutoSparse as KAutoSparse, + AutoSparse as KerasAutoSparse, ) from pquant.core.keras.pruning_methods.cs import ( # noqa: E402 - ContinuousSparsification as KCS, + ContinuousSparsification as KerasContinuousSparsification, ) -from pquant.core.keras.pruning_methods.dst import DST as KDST # noqa: E402 -from pquant.core.keras.pruning_methods.mdmm import MDMM as KMDMM # noqa: E402 +from pquant.core.keras.pruning_methods.dst import DST as KerasDST # noqa: E402 +from pquant.core.keras.pruning_methods.mdmm import MDMM as KerasMDMM # noqa: E402 from pquant.core.keras.pruning_methods.metric_functions import ( # noqa: E402 - StructuredSparsityMetric as KStructuredSparsityMetric, + StructuredSparsityMetric as KerasStructuredSparsityMetric, ) from pquant.core.keras.pruning_methods.metric_functions import ( # noqa: E402 - UnstructuredSparsityMetric as KUnstructuredSparsityMetric, + UnstructuredSparsityMetric as KerasUnstructuredSparsityMetric, ) -from pquant.core.keras.pruning_methods.pdp import PDP as KPDP # noqa: E402 -from pquant.core.keras.pruning_methods.wanda import Wanda as KWanda # noqa: E402 +from pquant.core.keras.pruning_methods.pdp import PDP as KerasPDP # noqa: E402 +from pquant.core.keras.pruning_methods.wanda import Wanda as KerasWanda # noqa: E402 from pquant.core.torch.pruning_methods.activation_pruning import ( # noqa: E402 - ActivationPruning as TActivationPruning, + ActivationPruning as TorchActivationPruning, ) from pquant.core.torch.pruning_methods.autosparse import ( # noqa: E402 - AutoSparse as TAutoSparse, + AutoSparse as TorchAutoSparse, ) from pquant.core.torch.pruning_methods.cs import ( # noqa: E402 - ContinuousSparsification as TCS, + ContinuousSparsification as TorchContinuousSparsification, ) -from pquant.core.torch.pruning_methods.dst import DST as TDST # noqa: E402 -from pquant.core.torch.pruning_methods.mdmm import MDMM as TMDMM # noqa: E402 +from pquant.core.torch.pruning_methods.dst import DST as TorchDST # noqa: E402 +from pquant.core.torch.pruning_methods.mdmm import MDMM as TorchMDMM # noqa: E402 from pquant.core.torch.pruning_methods.metric_functions import ( # noqa: E402 - StructuredSparsityMetric as TStructuredSparsityMetric, + StructuredSparsityMetric as TorchStructuredSparsityMetric, ) from pquant.core.torch.pruning_methods.metric_functions import ( # noqa: E402 - UnstructuredSparsityMetric as TUnstructuredSparsityMetric, + UnstructuredSparsityMetric as TorchUnstructuredSparsityMetric, ) -from pquant.core.torch.pruning_methods.pdp import PDP as TPDP # noqa: E402 -from pquant.core.torch.pruning_methods.wanda import Wanda as TWanda # noqa: E402 +from pquant.core.torch.pruning_methods.pdp import PDP as TorchPDP # noqa: E402 +from pquant.core.torch.pruning_methods.wanda import Wanda as TorchWanda # noqa: E402 -ATOL = 1e-5 -RTOL = 1e-4 +ABSOLUTE_TOLERANCE = 1e-5 +RELATIVE_TOLERANCE = 1e-4 -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _to_numpy(x): +def to_numpy(x): if isinstance(x, torch.Tensor): return x.detach().cpu().numpy() return np.asarray(ops.convert_to_numpy(x)) -def _assert_close(a, b, atol=ATOL, rtol=RTOL, msg=""): - a_np = _to_numpy(a) - b_np = _to_numpy(b) +def assert_close(a, b, atol=ABSOLUTE_TOLERANCE, rtol=RELATIVE_TOLERANCE, msg=""): + a_np = to_numpy(a) + b_np = to_numpy(b) assert a_np.shape == b_np.shape, f"{msg}: shape mismatch: {a_np.shape} vs {b_np.shape}" np.testing.assert_allclose(a_np, b_np, atol=atol, rtol=rtol, err_msg=msg) -def _keras_tensor(arr): +def keras_tensor(arr): return ops.convert_to_tensor(np.asarray(arr).astype(np.float32)) -def _torch_tensor(arr): - return torch.as_tensor(np.asarray(arr).astype(np.float32)) +def torch_tensor(arr, requires_grad=False): + return torch.as_tensor(np.asarray(arr).astype(np.float32)).requires_grad_(requires_grad) -def _reset_seed(seed=0): +def reset_seed(seed=0): np.random.seed(seed) torch.manual_seed(seed) -# --------------------------------------------------------------------------- -# ActivationPruning -# --------------------------------------------------------------------------- +def keras_grad(fn, x): + """Gradient of scalar-reducing ``fn(x).sum()`` w.r.t. keras tensor ``x``.""" + import keras + + if keras.backend.backend() == "tensorflow": + import tensorflow as tf + xt = tf.convert_to_tensor(x) + with tf.GradientTape() as tape: + tape.watch(xt) + loss = ops.sum(fn(xt)) + return tape.gradient(loss, xt) + raise RuntimeError("keras_grad only supports the tensorflow backend") -def _ap_config(): + +def ap_config(): return { "pruning_parameters": { "pruning_method": "activation_pruning", @@ -121,11 +126,11 @@ def _ap_config(): ], ) def test_activation_pruning_matches_keras(layer_type, shape): - cfg = _ap_config() + cfg = ap_config() out_channels = shape[0] batch = 32 - _reset_seed() + reset_seed() weight_np = np.random.randn(*shape).astype(np.float32) # Construct outputs with distinct per-channel activity levels so the # resulting mask is non-trivial (some channels pct_active > threshold, @@ -138,22 +143,22 @@ def test_activation_pruning_matches_keras(layer_type, shape): else: output_np = np.tile(per_channel[None, :, None, None], (batch, 1, 4, 4)) - k = KActivationPruning(cfg, layer_type) - k.build(shape) - k.post_pre_train_function() + k_layer = KerasActivationPruning(cfg, layer_type) + k_layer.build(shape) + k_layer.post_pre_train_function() - t = TActivationPruning(cfg, layer_type) - t.build(shape) - t.post_pre_train_function() + t_layer = TorchActivationPruning(cfg, layer_type) + t_layer.build(shape) + t_layer.post_pre_train_function() for _ in range(cfg["pruning_parameters"]["t_delta"]): - k.collect_output(_keras_tensor(output_np), training=True) - t.collect_output(_torch_tensor(output_np), training=True) + k_layer.collect_output(keras_tensor(output_np), training=True) + t_layer.collect_output(torch_tensor(output_np), training=True) - k.post_epoch_function(0, 1) - t.post_epoch_function(0, 1) + k_layer.post_epoch_function(0, 1) + t_layer.post_epoch_function(0, 1) - _assert_close(k.mask, t.mask, msg=f"AP mask ({layer_type})") + assert_close(k_layer.mask, t_layer.mask, msg=f"AP mask ({layer_type})") # Sanity check: the constructed per-channel outputs put ~1/3 of the # channels at non-positive values, so their pct_active == 0 falls below @@ -161,22 +166,23 @@ def test_activation_pruning_matches_keras(layer_type, shape): # value hit pct_active == 1 and survive. The expected pruned count is # deterministic from per_channel — verifying we actually exercise both # branches rather than matching a trivial all-ones mask. - mask_np = _to_numpy(t.mask) + mask_np = to_numpy(t_layer.mask) pruned_fraction = float((mask_np == 0).sum()) / mask_np.size # linspace(-0.5, 1.0, 16) → 6 values <= 0 → 6/16 = 0.375 pruned channels. assert pruned_fraction == pytest.approx(0.375), f"AP mask ({layer_type}) pruned fraction {pruned_fraction} != 0.375" - k_out = k(_keras_tensor(weight_np)) - t_out = t(_torch_tensor(weight_np)) - _assert_close(k_out, t_out, msg=f"AP forward ({layer_type})") - + k_weight = keras_tensor(weight_np) + t_weight = torch_tensor(weight_np, requires_grad=True) + k_out = k_layer(k_weight) + t_out = t_layer(t_weight) + assert_close(k_out, t_out, msg=f"AP forward ({layer_type})") -# --------------------------------------------------------------------------- -# PDP -# --------------------------------------------------------------------------- + k_grad = keras_grad(k_layer, weight_np) + t_out.sum().backward() + assert_close(k_grad, t_weight.grad, msg=f"AP backward ({layer_type})") -def _pdp_config(sparsity=0.75, structured=False): +def pdp_config(sparsity=0.75, structured=False): return { "pruning_parameters": { "pruning_method": "pdp", @@ -201,52 +207,53 @@ def _pdp_config(sparsity=0.75, structured=False): ], ) def test_pdp_matches_keras(layer_type, shape, structured): - cfg = _pdp_config(structured=structured) + cfg = pdp_config(structured=structured) target_sparsity = cfg["pruning_parameters"]["sparsity"] - _reset_seed() + reset_seed() weight_np = np.random.randn(*shape).astype(np.float32) - k = KPDP(cfg, layer_type) - k.build(shape) - k.post_pre_train_function() + k_layer = KerasPDP(cfg, layer_type) + k_layer.build(shape) + k_layer.post_pre_train_function() - t = TPDP(cfg, layer_type) - t.build(shape) - t.post_pre_train_function() + t_layer = TorchPDP(cfg, layer_type) + t_layer.build(shape) + t_layer.post_pre_train_function() # Force the sparsity ramp to be fully complete. pre_epoch_function sets # r = min(1, epsilon * (epoch + 1)) * init_r; with epsilon=1.0 that already # puts the ramp multiplier at 1.0 on epoch 0, so r = init_r = 0.75. - k.pre_epoch_function(0, None) - t.pre_epoch_function(0, None) + k_layer.pre_epoch_function(0, None) + t_layer.pre_epoch_function(0, None) + + k_weight = keras_tensor(weight_np) + t_weight = torch_tensor(weight_np, requires_grad=True) + k_out = k_layer(k_weight) + t_out = t_layer(t_weight) + assert_close(k_out, t_out, msg=f"PDP forward ({layer_type}, structured={structured})") - k_out = k(_keras_tensor(weight_np)) - t_out = t(_torch_tensor(weight_np)) - _assert_close(k_out, t_out, msg=f"PDP forward ({layer_type}, structured={structured})") + k_grad = keras_grad(k_layer, weight_np) + t_out.sum().backward() + assert_close(k_grad, t_weight.grad, msg=f"PDP backward ({layer_type}, structured={structured})") - k.update_mask(_keras_tensor(weight_np)) - t.update_mask(_torch_tensor(weight_np)) - _assert_close(k.mask, t.mask, msg="PDP mask after update_mask") + k_layer.update_mask(keras_tensor(weight_np)) + t_layer.update_mask(torch_tensor(weight_np)) + assert_close(k_layer.mask, t_layer.mask, msg="PDP mask after update_mask") # Verify the produced mask hits the configured target sparsity. # For structured pruning the mask has shape (C, 1, ...) and encodes # per-channel keep/prune; its sparsity directly equals the channel-level # pruning fraction. For unstructured it's per-element. With temperature # 1e-5 the soft mask is effectively binary, so use >= 0.5 to discretize. - t_mask_np = _to_numpy(t.mask) + t_mask_np = to_numpy(t_layer.mask) actual_sparsity = float((t_mask_np < 0.5).sum()) / t_mask_np.size assert actual_sparsity == pytest.approx(target_sparsity, abs=1e-6), ( f"PDP {layer_type} (structured={structured}) mask sparsity " f"{actual_sparsity} != target {target_sparsity}" ) -# --------------------------------------------------------------------------- -# ContinuousSparsification -# --------------------------------------------------------------------------- - - -def _cs_config(threshold_decay=1e-4): +def cs_config(threshold_decay=1e-4): return { "pruning_parameters": { "pruning_method": "cs", @@ -264,43 +271,44 @@ def _cs_config(threshold_decay=1e-4): [(16, 8), (16, 8, 3, 3)], ) def test_cs_matches_keras(shape): - cfg = _cs_config() + cfg = cs_config() layer_type = "linear" if len(shape) == 2 else "conv" - _reset_seed() + reset_seed() s_override_np = (np.random.randn(*shape) * 0.5).astype(np.float32) weight_np = np.random.randn(*shape).astype(np.float32) - k = KCS(cfg, layer_type) - k.build(shape) - k.post_pre_train_function() - k.s.assign(_keras_tensor(s_override_np)) + k_layer = KerasContinuousSparsification(cfg, layer_type) + k_layer.build(shape) + k_layer.post_pre_train_function() + k_layer.s.assign(keras_tensor(s_override_np)) - t = TCS(cfg, layer_type) - t.build(shape) - t.post_pre_train_function() + t_layer = TorchContinuousSparsification(cfg, layer_type) + t_layer.build(shape) + t_layer.post_pre_train_function() with torch.no_grad(): - t.s.data.copy_(_torch_tensor(s_override_np)) + t_layer.s.data.copy_(torch_tensor(s_override_np)) - k_out = k(_keras_tensor(weight_np)) - t_out = t(_torch_tensor(weight_np)) - _assert_close(k_out, t_out, msg=f"CS forward ({layer_type})") + k_weight = keras_tensor(weight_np) + t_weight = torch_tensor(weight_np, requires_grad=True) + k_out = k_layer(k_weight) + t_out = t_layer(t_weight) + assert_close(k_out, t_out, msg=f"CS forward ({layer_type})") - _assert_close(k.get_hard_mask(), t.get_hard_mask(), msg="CS hard mask") - _assert_close(k.calculate_additional_loss(), t.calculate_additional_loss(), msg="CS additional loss") + k_grad = keras_grad(k_layer, weight_np) + t_out.sum().backward() + assert_close(k_grad, t_weight.grad, msg=f"CS backward ({layer_type})") - # post_epoch_function updates beta — trajectories should match. - k.post_epoch_function(0, 5) - t.post_epoch_function(0, 5) - _assert_close(k.beta, t.beta, msg="CS beta after post_epoch_function") + assert_close(k_layer.get_hard_mask(), t_layer.get_hard_mask(), msg="CS hard mask") + assert_close(k_layer.calculate_additional_loss(), t_layer.calculate_additional_loss(), msg="CS additional loss") - -# --------------------------------------------------------------------------- -# DST -# --------------------------------------------------------------------------- + # post_epoch_function updates beta — trajectories should match. + k_layer.post_epoch_function(0, 5) + t_layer.post_epoch_function(0, 5) + assert_close(k_layer.beta, t_layer.beta, msg="CS beta after post_epoch_function") -def _dst_config(threshold_type="channelwise"): +def dst_config(threshold_type="channelwise"): return { "pruning_parameters": { "pruning_method": "dst", @@ -325,9 +333,9 @@ def _dst_config(threshold_type="channelwise"): ], ) def test_dst_matches_keras(layer_type, shape, threshold_type): - cfg = _dst_config(threshold_type=threshold_type) + cfg = dst_config(threshold_type=threshold_type) - _reset_seed() + reset_seed() weight_np = (np.random.randn(*shape) * 0.5).astype(np.float32) if threshold_type == "layerwise": thr_np = np.array([[0.1]], dtype=np.float32) @@ -336,35 +344,36 @@ def test_dst_matches_keras(layer_type, shape, threshold_type): else: # weightwise thr_np = (np.random.rand(shape[0], int(np.prod(shape[1:]))) * 0.2).astype(np.float32) - k = KDST(cfg, layer_type) - k.build(shape) - k.post_pre_train_function() - k.threshold.assign(_keras_tensor(thr_np)) + k_layer = KerasDST(cfg, layer_type) + k_layer.build(shape) + k_layer.post_pre_train_function() + k_layer.threshold.assign(keras_tensor(thr_np)) - t = TDST(cfg, layer_type) - t.build(shape) - t.post_pre_train_function() + t_layer = TorchDST(cfg, layer_type) + t_layer.build(shape) + t_layer.post_pre_train_function() with torch.no_grad(): - t.threshold.data.copy_(_torch_tensor(thr_np)) + t_layer.threshold.data.copy_(torch_tensor(thr_np)) - k_out = k(_keras_tensor(weight_np)) - t_out = t(_torch_tensor(weight_np)) - _assert_close(k_out, t_out, msg=f"DST forward ({layer_type}, {threshold_type})") + k_weight = keras_tensor(weight_np) + t_weight = torch_tensor(weight_np, requires_grad=True) + k_out = k_layer(k_weight) + t_out = t_layer(t_weight) + assert_close(k_out, t_out, msg=f"DST forward ({layer_type}, {threshold_type})") - _assert_close( - k.get_mask(_keras_tensor(weight_np)), - t.get_mask(_torch_tensor(weight_np)), + k_grad = keras_grad(k_layer, weight_np) + t_out.sum().backward() + assert_close(k_grad, t_weight.grad, msg=f"DST backward ({layer_type}, {threshold_type})") + + assert_close( + k_layer.get_mask(keras_tensor(weight_np)), + t_layer.get_mask(torch_tensor(weight_np)), msg=f"DST get_mask ({threshold_type})", ) - _assert_close(k.calculate_additional_loss(), t.calculate_additional_loss(), msg="DST additional loss") - - -# --------------------------------------------------------------------------- -# Wanda -# --------------------------------------------------------------------------- + assert_close(k_layer.calculate_additional_loss(), t_layer.calculate_additional_loss(), msg="DST additional loss") -def _wanda_config(sparsity=0.75, N=None, M=None): +def wanda_config(sparsity=0.75, N=None, M=None): return { "pruning_parameters": { "pruning_method": "wanda", @@ -391,51 +400,52 @@ def _wanda_config(sparsity=0.75, N=None, M=None): ], ) def test_wanda_matches_keras(layer_type, shape, N, M): - cfg = _wanda_config(N=N, M=M) + cfg = wanda_config(N=N, M=M) - _reset_seed() + reset_seed() if layer_type == "linear": x_np = np.random.randn(32, shape[1]).astype(np.float32) else: x_np = np.random.randn(32, shape[1], shape[2], shape[3]).astype(np.float32) w_np = np.random.randn(*shape).astype(np.float32) - k = KWanda(cfg, layer_type) - k.build(shape) - k.post_pre_train_function() + k_layer = KerasWanda(cfg, layer_type) + k_layer.build(shape) + k_layer.post_pre_train_function() - t = TWanda(cfg, layer_type) - t.build(shape) - t.post_pre_train_function() + t_layer = TorchWanda(cfg, layer_type) + t_layer.build(shape) + t_layer.post_pre_train_function() for _ in range(cfg["pruning_parameters"]["t_delta"]): - k.collect_input(_keras_tensor(x_np), _keras_tensor(w_np), training=True) - t.collect_input(_torch_tensor(x_np), _torch_tensor(w_np), training=True) + k_layer.collect_input(keras_tensor(x_np), keras_tensor(w_np), training=True) + t_layer.collect_input(torch_tensor(x_np), torch_tensor(w_np), training=True) - _assert_close(k.mask, t.mask, msg=f"Wanda mask ({layer_type}, N={N}, M={M})") + assert_close(k_layer.mask, t_layer.mask, msg=f"Wanda mask ({layer_type}, N={N}, M={M})") # Verify the mask hits the configured target sparsity. For N:M pruning # Wanda internally uses N/M as the sparsity target; for unstructured it # uses the configured sparsity directly. Mask values are strictly {0, 1} # (produced by topk + scatter), so `== 0` counts pruned entries. target_sparsity = (N / M) if (N is not None and M is not None) else cfg["pruning_parameters"]["sparsity"] - mask_np = _to_numpy(t.mask) + mask_np = to_numpy(t_layer.mask) pruned_fraction = float((mask_np == 0).sum()) / mask_np.size assert pruned_fraction == pytest.approx( target_sparsity ), f"Wanda {layer_type} (N={N}, M={M}) pruned fraction {pruned_fraction} != target {target_sparsity}" - k_out = k(_keras_tensor(w_np)) - t_out = t(_torch_tensor(w_np)) - _assert_close(k_out, t_out, msg=f"Wanda forward ({layer_type}, N={N}, M={M})") + k_weight = keras_tensor(w_np) + t_weight = torch_tensor(w_np, requires_grad=True) + k_out = k_layer(k_weight) + t_out = t_layer(t_weight) + assert_close(k_out, t_out, msg=f"Wanda forward ({layer_type}, N={N}, M={M})") + k_grad = keras_grad(k_layer, w_np) + t_out.sum().backward() + assert_close(k_grad, t_weight.grad, msg=f"Wanda backward ({layer_type}, N={N}, M={M})") -# --------------------------------------------------------------------------- -# AutoSparse -# --------------------------------------------------------------------------- - -def _autosparse_config(threshold_type="channelwise", threshold_init=-2.0): +def autosparse_config(threshold_type="channelwise", threshold_init=-2.0): return { "pruning_parameters": { "pruning_method": "autosparse", @@ -470,43 +480,44 @@ def test_autosparse_matches_keras(layer_type, shape, threshold_type): if _keras.backend.backend() == "torch": pytest.skip("Keras AutoSparse forward is incompatible with the torch backend.") - cfg = _autosparse_config(threshold_type=threshold_type) + cfg = autosparse_config(threshold_type=threshold_type) - _reset_seed() + reset_seed() weight_np = np.random.randn(*shape).astype(np.float32) - k = KAutoSparse(cfg, layer_type) - k.build(shape) - k.post_pre_train_function() + k_layer = KerasAutoSparse(cfg, layer_type) + k_layer.build(shape) + k_layer.post_pre_train_function() - t = TAutoSparse(cfg, layer_type) - t.build(shape) - t.post_pre_train_function() + t_layer = TorchAutoSparse(cfg, layer_type) + t_layer.build(shape) + t_layer.post_pre_train_function() with torch.no_grad(): - t.threshold.data.copy_(_torch_tensor(_to_numpy(k.threshold))) + t_layer.threshold.data.copy_(torch_tensor(to_numpy(k_layer.threshold))) - _assert_close( - k.get_mask(_keras_tensor(weight_np)), - t.get_mask(_torch_tensor(weight_np)), + assert_close( + k_layer.get_mask(keras_tensor(weight_np)), + t_layer.get_mask(torch_tensor(weight_np)), msg=f"AutoSparse get_mask ({layer_type}, {threshold_type})", ) - k_out = k(_keras_tensor(weight_np)) - t_out = t(_torch_tensor(weight_np)) - _assert_close(k_out, t_out, msg=f"AutoSparse forward ({layer_type}, {threshold_type})") - - # post_epoch_function updates alpha via decay; trajectories should match. - k.post_epoch_function(3, 10) - t.post_epoch_function(3, 10) - _assert_close(k.alpha, t.alpha, msg="AutoSparse alpha after post_epoch_function") + k_weight = keras_tensor(weight_np) + t_weight = torch_tensor(weight_np, requires_grad=True) + k_out = k_layer(k_weight) + t_out = t_layer(t_weight) + assert_close(k_out, t_out, msg=f"AutoSparse forward ({layer_type}, {threshold_type})") + k_grad = keras_grad(k_layer, weight_np) + t_out.sum().backward() + assert_close(k_grad, t_weight.grad, msg=f"AutoSparse backward ({layer_type}, {threshold_type})") -# --------------------------------------------------------------------------- -# MDMM -# --------------------------------------------------------------------------- + # post_epoch_function updates alpha via decay; trajectories should match. + k_layer.post_epoch_function(3, 10) + t_layer.post_epoch_function(3, 10) + assert_close(k_layer.alpha, t_layer.alpha, msg="AutoSparse alpha after post_epoch_function") -def _mdmm_config( +def mdmm_config( constraint_type="Equality", metric_type="UnstructuredSparsity", target_value=0.5, @@ -544,83 +555,84 @@ def _mdmm_config( ], ) def test_mdmm_matches_keras(constraint_type, metric_type): - cfg = _mdmm_config(constraint_type=constraint_type, metric_type=metric_type) + cfg = mdmm_config(constraint_type=constraint_type, metric_type=metric_type) shape = (16, 8) - _reset_seed() + reset_seed() weight_np = (np.random.randn(*shape) * 0.2).astype(np.float32) - k = KMDMM(cfg, "linear") - k.build(shape) - k.post_pre_train_function() + k_layer = KerasMDMM(cfg, "linear") + k_layer.build(shape) + k_layer.post_pre_train_function() + + t_layer = TorchMDMM(cfg, "linear") + t_layer.build(shape) + t_layer.post_pre_train_function() - t = TMDMM(cfg, "linear") - t.build(shape) - t.post_pre_train_function() + k_weight = keras_tensor(weight_np) + t_weight = torch_tensor(weight_np, requires_grad=True) + k_out = k_layer(k_weight) + t_out = t_layer(t_weight) + assert_close(k_out, t_out, msg=f"MDMM forward ({constraint_type}, {metric_type})") - k_out = k(_keras_tensor(weight_np)) - t_out = t(_torch_tensor(weight_np)) - _assert_close(k_out, t_out, msg=f"MDMM forward ({constraint_type}, {metric_type})") + k_grad = keras_grad(k_layer, weight_np) + t_out.sum().backward() + assert_close(k_grad, t_weight.grad, msg=f"MDMM backward ({constraint_type}, {metric_type})") - _assert_close( - k.get_hard_mask(_keras_tensor(weight_np)), - t.get_hard_mask(_torch_tensor(weight_np)), + assert_close( + k_layer.get_hard_mask(keras_tensor(weight_np)), + t_layer.get_hard_mask(torch_tensor(weight_np)), msg="MDMM hard_mask", ) # Constraint penalty: read directly from the constraint layer to avoid # differences in how keras/torch surface accumulated losses. - k_penalty = ops.sum(k.constraint_layer(_keras_tensor(weight_np))) - t_penalty = t.constraint_layer(_torch_tensor(weight_np)).sum() - _assert_close(k_penalty, t_penalty, msg="MDMM constraint penalty") + k_penalty = ops.sum(k_layer.constraint_layer(keras_tensor(weight_np))) + t_penalty = t_layer.constraint_layer(torch_tensor(weight_np)).sum() + assert_close(k_penalty, t_penalty, msg="MDMM constraint penalty") def test_mdmm_finetune_returns_masked_weight(): """In finetuning mode both layers should return weight * hard_mask.""" - cfg = _mdmm_config() + cfg = mdmm_config() shape = (8, 6) - _reset_seed() + reset_seed() weight_np = (np.random.randn(*shape) * 0.2).astype(np.float32) - k = KMDMM(cfg, "linear") - k.build(shape) - k.post_pre_train_function() - k.pre_finetune_function() - - t = TMDMM(cfg, "linear") - t.build(shape) - t.post_pre_train_function() - t.pre_finetune_function() - - k_out = k(_keras_tensor(weight_np)) - t_out = t(_torch_tensor(weight_np)) - _assert_close(k_out, t_out, msg="MDMM finetune forward") + k_layer = KerasMDMM(cfg, "linear") + k_layer.build(shape) + k_layer.post_pre_train_function() + k_layer.pre_finetune_function() + t_layer = TorchMDMM(cfg, "linear") + t_layer.build(shape) + t_layer.post_pre_train_function() + t_layer.pre_finetune_function() -# --------------------------------------------------------------------------- -# Metric functions -# --------------------------------------------------------------------------- + k_out = k_layer(keras_tensor(weight_np)) + t_out = t_layer(torch_tensor(weight_np)) + assert_close(k_out, t_out, msg="MDMM finetune forward") @pytest.mark.parametrize("l0_mode", ["coarse", "smooth"]) @pytest.mark.parametrize("scale_mode", ["mean", "sum"]) def test_unstructured_sparsity_metric_matches_keras(l0_mode, scale_mode): - k = KUnstructuredSparsityMetric(l0_mode=l0_mode, scale_mode=scale_mode, target_sparsity=0.7, epsilon=1e-3) - t = TUnstructuredSparsityMetric(l0_mode=l0_mode, scale_mode=scale_mode, target_sparsity=0.7, epsilon=1e-3) + k_layer = KerasUnstructuredSparsityMetric(l0_mode=l0_mode, scale_mode=scale_mode, target_sparsity=0.7, epsilon=1e-3) + t_layer = TorchUnstructuredSparsityMetric(l0_mode=l0_mode, scale_mode=scale_mode, target_sparsity=0.7, epsilon=1e-3) - _reset_seed() + reset_seed() w_np = (np.random.randn(16, 8) * 0.1).astype(np.float32) - _assert_close(k(_keras_tensor(w_np)), t(_torch_tensor(w_np)), msg=f"Unstructured({l0_mode},{scale_mode})") + assert_close(k_layer(keras_tensor(w_np)), t_layer(torch_tensor(w_np)), msg=f"Unstructured({l0_mode},{scale_mode})") @pytest.mark.parametrize("rf", [1, 4, 5]) def test_structured_sparsity_metric_matches_keras(rf): - k = KStructuredSparsityMetric(rf=rf, epsilon=1e-3) - t = TStructuredSparsityMetric(rf=rf, epsilon=1e-3) + k_layer = KerasStructuredSparsityMetric(rf=rf, epsilon=1e-3) + t_layer = TorchStructuredSparsityMetric(rf=rf, epsilon=1e-3) - _reset_seed() + reset_seed() w_np = (np.random.randn(12, 7) * 0.05).astype(np.float32) - _assert_close(k(_keras_tensor(w_np)), t(_torch_tensor(w_np)), msg=f"Structured(rf={rf})") + assert_close(k_layer(keras_tensor(w_np)), t_layer(torch_tensor(w_np)), msg=f"Structured(rf={rf})") From d874f0d02a32eedd6895a14f59676c178921b754 Mon Sep 17 00:00:00 2001 From: Roope Niemi Date: Thu, 30 Jul 2026 13:51:09 +0200 Subject: [PATCH 18/22] update pre-commit --- .pre-commit-config.yaml | 38 ++++++++++++++++++-------------------- pyproject.toml | 13 ++++++++----- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dbee399..3192c21 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,19 +1,20 @@ +exclude: (^examples\/) repos: -- repo: https://github.com/psf/black - rev: 25.1.0 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.20 hooks: - - id: black - language_version: python3 - args: ['--line-length=125', - '--skip-string-normalization'] + - id: ruff + args: [--fix] + - id: ruff-format - repo: https://github.com/tox-dev/pyproject-fmt - rev: v2.5.1 + rev: v2.25.1 hooks: - id: pyproject-fmt + args: ["--max-supported-python", "3.13"] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -27,29 +28,26 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace -- repo: https://github.com/PyCQA/isort - rev: 6.0.1 - hooks: - - id: isort - args: ["--profile=black"] # <-- this one - - repo: https://github.com/asottile/pyupgrade - rev: v3.19.1 + rev: v3.21.2 hooks: - id: pyupgrade - args: ["--py36-plus"] + args: ["--py310-plus"] - repo: https://github.com/pycqa/flake8 - rev: 7.1.2 + rev: 7.3.0 hooks: - id: flake8 - exclude: docs/conf.py + exclude: docs/source/conf.py additional_dependencies: [flake8-bugbear, flake8-print] args: ['--max-line-length=125', # github viewer width - '--extend-ignore=E203'] # E203 is not PEP8 compliant + '--extend-ignore=E203,T201,F401', + # E203 is not PEP8 compliant + # F401 included in ruff (behaves slightly differently for noqa flags) + ] - repo: https://github.com/mgedmin/check-manifest - rev: "0.50" + rev: "0.51" hooks: - id: check-manifest stages: [manual] diff --git a/pyproject.toml b/pyproject.toml index aaa2530..58af8da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,5 @@ [build-system] build-backend = "setuptools.build_meta" - requires = [ "setuptools>=46.1", "setuptools-scm[toml]>=5" ] [project] @@ -29,14 +28,18 @@ optional-dependencies.tensorflow = [ "tensorflow>=2.17,<=2.20" ] optional-dependencies.test = [ "pytest>=8.4" ] optional-dependencies.torch = [ "torch>=2.1" ] urls.repository = "https://github.com/cern-nextgen/PQuantML" - -entry-points."alkaid_keras".pquant = "pquant._alkaid_plugin._alkaid_keras_plugin:register" -entry-points."alkaid_torch".pquant = "pquant._alkaid_plugin._alkaid_torch_plugin:register" +entry-points.alkaid_keras.pquant = "pquant._alkaid_plugin._alkaid_keras_plugin:register" +entry-points.alkaid_torch.pquant = "pquant._alkaid_plugin._alkaid_torch_plugin:register" [tool.setuptools] packages = [ "pquant" ] -include-package-data = true package-dir = { "" = "src" } +include-package-data = true [tool.setuptools_scm] write_to = "src/pquant/_version.py" + +[tool.ruff] +line-length = 125 +src = [ "src" ] +lint.extend-select = [ "I" ] From eb65ffb809108b784e2051767f0c0f3f9e672fb3 Mon Sep 17 00:00:00 2001 From: Roope Niemi Date: Thu, 30 Jul 2026 16:51:51 +0200 Subject: [PATCH 19/22] ruff refactors --- README.md | 2 +- docs/source/conf.py | 36 +++---- docs/source/index.rst | 6 +- docs/source/reference.md | 2 +- src/pquant/_alkaid_plugin/_alkaid_common.py | 62 ++++++------ .../_alkaid_plugin/_alkaid_keras_plugin.py | 36 +++---- .../_alkaid_plugin/_alkaid_torch_plugin.py | 40 ++++---- src/pquant/core/constants.py | 2 +- .../core/hyperparameter_optimization.py | 47 ++++----- src/pquant/core/keras/activations.py | 20 ++-- src/pquant/core/keras/convert_to_onnx.py | 4 +- src/pquant/core/keras/layers.py | 64 ++++++------ .../pruning_methods/constraint_functions.py | 8 +- .../keras/pruning_methods/metric_functions.py | 14 +-- src/pquant/core/torch/activations.py | 20 ++-- src/pquant/core/torch/convert_to_onnx.py | 8 +- src/pquant/core/torch/distillers.py | 18 ++-- src/pquant/core/torch/fit_compress.py | 47 +++------ .../core/torch/fixed_point_quantizer.py | 46 ++++----- src/pquant/core/torch/layers.py | 99 +++++++++---------- src/pquant/core/torch/optimizers.py | 16 +-- .../torch/pruning_methods/metric_functions.py | 10 +- src/pquant/data_models/fitcompress_model.py | 4 +- .../hyperparameter_optimization_model.py | 8 +- src/pquant/data_models/pruning_model.py | 8 +- src/pquant/data_models/training_model.py | 2 +- tests/conftest.py | 18 ++-- tests/test_hgq_keras.py | 6 +- tests/test_hgq_torch.py | 8 +- tests/test_keras_alkaid_conversion.py | 8 +- tests/test_keras_compression_layers.py | 61 ++++++------ tests/test_torch_alkaid_conversion.py | 8 +- tests/test_torch_checkpoint.py | 4 +- tests/test_torch_compression_layers.py | 32 +++--- tests/test_torch_missing_quantizer_tracing.py | 3 +- tests/test_torch_onnx_converter.py | 13 +-- tests/test_torch_pruning_layers.py | 8 +- 37 files changed, 388 insertions(+), 410 deletions(-) diff --git a/README.md b/README.md index fc3a782..ed4f679 100644 --- a/README.md +++ b/README.md @@ -115,4 +115,4 @@ If you use PQuantML in your work, please cite: - Mia Liu (Purdue University) - Michael Kagan (SLAC National Accelerator Laboratory) - Vladimir Loncar (CERN) -- Maurizio Pierini (CERN) \ No newline at end of file +- Maurizio Pierini (CERN) diff --git a/docs/source/conf.py b/docs/source/conf.py index d439e8c..971c162 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -9,11 +9,11 @@ import os import sys -sys.path.insert(0, os.path.abspath('../')) +sys.path.insert(0, os.path.abspath("../")) -project = 'PQuantML' -copyright = '2025, Roope Niemi' -author = 'Roope Niemi, Anastasiia Petrovych' +project = "PQuantML" +copyright = "2025, Roope Niemi" +author = "Roope Niemi, Anastasiia Petrovych" release = "0.0.6" version = release @@ -35,32 +35,32 @@ autosummary_generate = True -extensions = ['myst_parser', 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.napoleon', 'sphinx_rtd_theme'] +extensions = ["myst_parser", "sphinx.ext.autodoc", "sphinx.ext.autosummary", "sphinx.ext.napoleon", "sphinx_rtd_theme"] -source_suffix = ['.rst', '.md'] +source_suffix = [".rst", ".md"] -templates_path = ['_templates'] -exclude_patterns = ['_build'] +templates_path = ["_templates"] +exclude_patterns = ["_build"] html_logo = "_static/pquant.png" html_theme_options = { - 'logo_only': True, - 'display_version': True, + "logo_only": True, + "display_version": True, } html_context = { - 'display_github': True, # Integrate GitHub - 'github_user': 'nroope', # Username - 'github_repo': "PQuant", # Repo name - 'github_version': 'master', # Version - 'conf_py_path': '/docs/', # Path in the checkout to the docs root + "display_github": True, # Integrate GitHub + "github_user": "nroope", # Username + "github_repo": "PQuant", # Repo name + "github_version": "master", # Version + "conf_py_path": "/docs/", # Path in the checkout to the docs root } html_theme = "sphinx_rtd_theme" -html_static_path = ['_static'] -html_favicon = '_static/pquant.png' +html_static_path = ["_static"] +html_favicon = "_static/pquant.png" html_css_files = [ - 'custom.css', + "custom.css", ] diff --git a/docs/source/index.rst b/docs/source/index.rst index ae6d779..5348005 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -20,13 +20,13 @@ Welcome to the documentation for PQuantML, a hardware-aware model compression fr - Layer-wise precision configuration - Flexible training pipelines - PyTorch and TensorFlow backends -- Knowledge distillation +- Knowledge distillation - HGQ library intergration - Integration with hardware-friendly toolchains (e.g., hls4ml) PQuantML enables efficient deployment of compact neural networks on resource-constrained hardware such as FPGAs and embedded accelerators. -The paper describing the framework is available at: `PQuantML: A Tool for End-to-End Hardware-aware Model Compression `_. +The paper describing the framework is available at: `PQuantML: A Tool for End-to-End Hardware-aware Model Compression `_. .. rst-class:: light @@ -65,4 +65,4 @@ Indices and tables ================== * :ref:`genindex` -* :ref:`search` \ No newline at end of file +* :ref:`search` diff --git a/docs/source/reference.md b/docs/source/reference.md index e74cc74..53aafef 100644 --- a/docs/source/reference.md +++ b/docs/source/reference.md @@ -218,4 +218,4 @@ PQuantML supports two quantization modes, each with several granularity options: **HGQ (High Granularity Quantization)**: - per-weight (learned bit-widths per weight) - per-tensor (learned bit-widths per tensor) -``` \ No newline at end of file +``` diff --git a/src/pquant/_alkaid_plugin/_alkaid_common.py b/src/pquant/_alkaid_plugin/_alkaid_common.py index 77b64e4..9280b8a 100644 --- a/src/pquant/_alkaid_plugin/_alkaid_common.py +++ b/src/pquant/_alkaid_plugin/_alkaid_common.py @@ -15,9 +15,9 @@ def _to_numpy(value: Any) -> np.ndarray: return np.array(0.0) if isinstance(value, np.ndarray): return value - if hasattr(value, 'detach'): + if hasattr(value, "detach"): value = value.detach() - if hasattr(value, 'cpu'): + if hasattr(value, "cpu"): value = value.cpu() return value.numpy() try: @@ -45,7 +45,7 @@ def _to_int_bits(value: Any) -> np.ndarray: def _raw_module_attr(obj: Any, name: str, default: Any = None) -> Any: - for storage_name in ('_parameters', '_buffers', '_modules'): + for storage_name in ("_parameters", "_buffers", "_modules"): storage = getattr(obj, storage_name, None) if isinstance(storage, dict) and name in storage: return storage[name] @@ -56,20 +56,20 @@ def _raw_module_attr(obj: Any, name: str, default: Any = None) -> Any: def _quantizer_kif(quantizer: Any) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - if hasattr(quantizer, '_parameters'): - if not bool(_raw_module_attr(quantizer, 'use_hgq', False)): + if hasattr(quantizer, "_parameters"): + if not bool(_raw_module_attr(quantizer, "use_hgq", False)): return ( - _to_int_bits(_raw_module_attr(quantizer, 'k')), - _to_int_bits(_raw_module_attr(quantizer, 'i')), - _to_int_bits(_raw_module_attr(quantizer, 'f')), + _to_int_bits(_raw_module_attr(quantizer, "k")), + _to_int_bits(_raw_module_attr(quantizer, "i")), + _to_int_bits(_raw_module_attr(quantizer, "f")), ) - inner = _raw_module_attr(quantizer, 'quantizer') - if hasattr(inner, '_parameters') or hasattr(inner, '_buffers'): - k = _raw_module_attr(inner, '_k') - i = _raw_module_attr(inner, '_i_raw', None) + inner = _raw_module_attr(quantizer, "quantizer") + if hasattr(inner, "_parameters") or hasattr(inner, "_buffers"): + k = _raw_module_attr(inner, "_k") + i = _raw_module_attr(inner, "_i_raw", None) if i is None: - i = _raw_module_attr(inner, '_i') - f = _raw_module_attr(inner, '_f') + i = _raw_module_attr(inner, "_i") + f = _raw_module_attr(inner, "_f") return _to_int_bits(k), _to_int_bits(i), _to_int_bits(f) k, i, f = quantizer.get_quantization_bits() return _to_int_bits(k), _to_int_bits(i), _to_int_bits(f) @@ -77,14 +77,14 @@ def _quantizer_kif(quantizer: Any) -> tuple[np.ndarray, np.ndarray, np.ndarray]: def _replay_quantizer(quantizer: Any, x: Any) -> Any: k, i, f = _quantizer_kif(quantizer) - inner = _raw_module_attr(quantizer, 'quantizer', None) - overflow = _raw_module_attr(quantizer, 'overflow', _raw_module_attr(inner, 'overflow_mode', 'WRAP')) - round_mode = _raw_module_attr(quantizer, 'round_mode', _raw_module_attr(inner, 'round_mode', 'TRN')) + inner = _raw_module_attr(quantizer, "quantizer", None) + overflow = _raw_module_attr(quantizer, "overflow", _raw_module_attr(inner, "overflow_mode", "WRAP")) + round_mode = _raw_module_attr(quantizer, "round_mode", _raw_module_attr(inner, "round_mode", "TRN")) return alkaid_quantize(x, k=k, i=i, f=f, overflow_mode=str(overflow).upper(), round_mode=str(round_mode).upper()) def _replay_quantizer_if_enabled(layer: Any, quantizer_name: str, x: Any, flag_name: str) -> Any: - if not bool(getattr(layer, 'enable_quantization', True)): + if not bool(getattr(layer, "enable_quantization", True)): return x if not bool(getattr(layer, flag_name, True)): return x @@ -93,16 +93,16 @@ def _replay_quantizer_if_enabled(layer: Any, quantizer_name: str, x: Any, flag_n def _assert_final_compression(layer: Any) -> None: - if not _to_bool(_raw_module_attr(layer, 'final_compression_done', False)): + if not _to_bool(_raw_module_attr(layer, "final_compression_done", False)): raise PQuantAlkaidError( - f'{type(layer).__name__} must have apply_final_compression() applied before Alkaid conversion.' + f"{type(layer).__name__} must have apply_final_compression() applied before Alkaid conversion." ) def _final_bias(layer: Any) -> np.ndarray: """The layer's final (compressed) bias as numpy, or a scalar zero when absent.""" _assert_final_compression(layer) - bias = getattr(layer, '_bias', None) + bias = getattr(layer, "_bias", None) if bias is None: return np.array(0.0) return _to_numpy(bias) @@ -111,10 +111,10 @@ def _final_bias(layer: Any) -> np.ndarray: def _scale_by_relu_multiplier(layer: Any, x: Any) -> Any: """Apply PQActivation's power-of-two ReLU multiplier, which is used only without HGQ.""" applies = ( - not _to_bool(getattr(layer, 'use_hgq', False)) - and _to_bool(getattr(layer, 'use_multiplier', False)) - and layer.activation_name == 'relu' - and hasattr(layer, 'multiplier') + not _to_bool(getattr(layer, "use_hgq", False)) + and _to_bool(getattr(layer, "use_multiplier", False)) + and layer.activation_name == "relu" + and hasattr(layer, "multiplier") ) if not applies: return x @@ -124,23 +124,23 @@ def _scale_by_relu_multiplier(layer: Any, x: Any) -> Any: def _replay_table(table: Any, x: Any, table_fn: Any) -> Any: """Replay a lookup-table activation: quantize input, apply the table, quantize output.""" if not (table.quantize_output and table.enable_quantization): - name = getattr(table, 'name', None) or type(table).__name__ - raise PQuantAlkaidError(f'PQSoftmax table {name!r} must have an enabled output quantizer for Alkaid conversion.') - x = _replay_quantizer_if_enabled(table, 'input_quantizer', x, 'quantize_input') + name = getattr(table, "name", None) or type(table).__name__ + raise PQuantAlkaidError(f"PQSoftmax table {name!r} must have an enabled output quantizer for Alkaid conversion.") + x = _replay_quantizer_if_enabled(table, "input_quantizer", x, "quantize_input") out = x.apply(table_fn(table)) return _replay_quantizer(table.output_quantizer, out) def _replay_softmax(layer: Any, inputs: Any, table_fn: Any) -> Any: """Replay PQSoftmax through its exp and inverse-sum lookup tables.""" - inputs = _replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + inputs = _replay_quantizer_if_enabled(layer, "input_quantizer", inputs, "quantize_input") if layer.stable: inputs = np.max(inputs, axis=layer.axes, keepdims=True) - inputs # type: ignore exponents = _replay_table(layer.exp_table, inputs, table_fn) sums = np.sum(exponents, axis=layer.axes, keepdims=True) inverse_sums = _replay_table(layer.inv_table, sums, table_fn) out = exponents * inverse_sums - return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + return _replay_quantizer_if_enabled(layer, "output_quantizer", out, "quantize_output") def _mark_plugin_loaded(framework: str) -> None: @@ -148,6 +148,6 @@ def _mark_plugin_loaded(framework: str) -> None: try: from alkaid.converter import _plugin_loader - _plugin_loader._LOADED.add(('pquant', framework)) + _plugin_loader._LOADED.add(("pquant", framework)) except Exception: pass diff --git a/src/pquant/_alkaid_plugin/_alkaid_keras_plugin.py b/src/pquant/_alkaid_plugin/_alkaid_keras_plugin.py index 716dc63..f848222 100644 --- a/src/pquant/_alkaid_plugin/_alkaid_keras_plugin.py +++ b/src/pquant/_alkaid_plugin/_alkaid_keras_plugin.py @@ -51,7 +51,7 @@ def _table_fn(table): fn = table.activation_function def apply_fn(v: np.ndarray) -> np.ndarray: - t = keras.ops.cast(keras.ops.convert_to_tensor(v), 'float32') + t = keras.ops.cast(keras.ops.convert_to_tensor(v), "float32") return np.asarray(keras.ops.convert_to_numpy(fn(t)), dtype=np.float64) return apply_fn @@ -80,9 +80,9 @@ class ReplayPQuantDense(ReplayOperationBase): def call(self, inputs: FVArray) -> FVArray: layer = self.op - inputs = _replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') - out = np.einsum('...c,cC->...C', inputs, _final_kernel(layer)) + _final_bias(layer) - return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + inputs = _replay_quantizer_if_enabled(layer, "input_quantizer", inputs, "quantize_input") + out = np.einsum("...c,cC->...C", inputs, _final_kernel(layer)) + _final_bias(layer) + return _replay_quantizer_if_enabled(layer, "output_quantizer", out, "quantize_output") class ReplayPQuantConv(ReplayOperationBase): @@ -90,7 +90,7 @@ class ReplayPQuantConv(ReplayOperationBase): def call(self, inputs: FVArray) -> FVArray: layer = self.op - inputs = _replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + inputs = _replay_quantizer_if_enabled(layer, "input_quantizer", inputs, "quantize_input") kernel = _final_kernel(layer) bias = _final_bias(layer) @@ -122,9 +122,9 @@ def call(self, inputs: FVArray) -> FVArray: ) if bias.shape != (): out = out + bias - if layer.data_format == 'channels_first': + if layer.data_format == "channels_first": out = np.moveaxis(out, -1, 1) # type: ignore - return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + return _replay_quantizer_if_enabled(layer, "output_quantizer", out, "quantize_output") class ReplayPQuantSeparableConv(ReplayOperationBase): @@ -158,7 +158,7 @@ def fused_scale_offset(self) -> tuple[np.ndarray, np.ndarray]: def call(self, inputs: FVArray, mask=None) -> FVArray: layer = self.op - inputs = _replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + inputs = _replay_quantizer_if_enabled(layer, "input_quantizer", inputs, "quantize_input") scale, offset = self.fused_scale_offset() shape = [1] * inputs.ndim axis = layer.axis if isinstance(layer.axis, (list, tuple)) else [layer.axis] @@ -179,9 +179,9 @@ class ReplayPQuantAvgPool(ReplayPool): def call(self, inputs: FVArray, mask: None = None) -> FVArray: layer = self.op - inputs = _replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + inputs = _replay_quantizer_if_enabled(layer, "input_quantizer", inputs, "quantize_input") out = super().call(inputs, mask=mask) - return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + return _replay_quantizer_if_enabled(layer, "output_quantizer", out, "quantize_output") class ReplayPQuantActivation(ReplayOperationBase): @@ -191,11 +191,11 @@ class ReplayPQuantActivation(ReplayOperationBase): def call(self, inputs: FVArray) -> FVArray: layer = self.op inputs = _scale_by_relu_multiplier(layer, inputs) - inputs = _replay_quantizer_if_enabled(layer, 'input_quantizer', inputs, 'quantize_input') + inputs = _replay_quantizer_if_enabled(layer, "input_quantizer", inputs, "quantize_input") if layer.activation_name not in keras_numpy_unary_map: - raise PQuantAlkaidError(f'Unsupported PQuant activation for Alkaid conversion: {layer.activation_name!r}') + raise PQuantAlkaidError(f"Unsupported PQuant activation for Alkaid conversion: {layer.activation_name!r}") out = keras_numpy_unary_map[layer.activation_name](inputs) - return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + return _replay_quantizer_if_enabled(layer, "output_quantizer", out, "quantize_output") class ReplayPQuantSoftmax(ReplayOperationBase): @@ -204,7 +204,7 @@ class ReplayPQuantSoftmax(ReplayOperationBase): def call(self, inputs: FVArray, mask=None) -> FVArray: if mask is not None: - raise PQuantAlkaidError('PQSoftmax masks are not supported in Alkaid conversion.') + raise PQuantAlkaidError("PQSoftmax masks are not supported in Alkaid conversion.") return _replay_softmax(self.op, inputs, _table_fn) @@ -215,7 +215,7 @@ class ReplayPQuantMultiheadAttention(ReplayOperationBase): def call(self, inputs, key_padding_mask=None, attn_mask=None, need_weights=True): layer = self.op if key_padding_mask is not None or attn_mask is not None: - raise PQuantAlkaidError('Attention masks are not supported in Alkaid conversion.') + raise PQuantAlkaidError("Attention masks are not supported in Alkaid conversion.") query, key, value = _unpack_query_key_value(inputs) batch_size, query_len = query.shape[0], query.shape[1] @@ -232,13 +232,13 @@ def call(self, inputs, key_padding_mask=None, attn_mask=None, need_weights=True) v = v.reshape(batch_size, key_len, num_heads, head_dim).transpose(0, 2, 1, 3) scale = float(np.float32(layer.scale)) - attn_scores = einsum('bhtd,bhsd->bhts', q, k) * scale + attn_scores = einsum("bhtd,bhsd->bhts", q, k) * scale # The softmax's own input/output quantizers handle the scores and the attention weights attn_weights = ReplayPQuantSoftmax(layer.softmax).call(attn_scores) # Weighted sum of values (dropout is an inference no-op): (B, H, T, head_dim) - out = einsum('bhts,bhsd->bhtd', attn_weights, v) + out = einsum("bhts,bhsd->bhtd", attn_weights, v) # Merge heads: (B, T, E) out = out.transpose(0, 2, 1, 3).reshape(batch_size, query_len, layer.embed_dim) @@ -252,4 +252,4 @@ def call(self, inputs, key_padding_mask=None, attn_mask=None, need_weights=True) def register() -> None: """Entry point for Alkaid's ``alkaid_keras`` second-level plugin group.""" - _mark_plugin_loaded('keras') + _mark_plugin_loaded("keras") diff --git a/src/pquant/_alkaid_plugin/_alkaid_torch_plugin.py b/src/pquant/_alkaid_plugin/_alkaid_torch_plugin.py index 4f24183..57f11e3 100644 --- a/src/pquant/_alkaid_plugin/_alkaid_torch_plugin.py +++ b/src/pquant/_alkaid_plugin/_alkaid_torch_plugin.py @@ -53,13 +53,13 @@ def _final_weight(layer: torch.nn.Module) -> np.ndarray: def _activation_numpy_fn(layer: PQActivation): """The numpy elementwise function for a named PQActivation (relu/tanh/gelu/...).""" name = layer.activation_name - fn = torch_numpy_unary_map.get(name) or torch_numpy_unary_map.get(name.replace('_', '')) + fn = torch_numpy_unary_map.get(name) or torch_numpy_unary_map.get(name.replace("_", "")) if fn is not None: return fn - if name == 'leaky_relu': - slope = float(getattr(layer.activation_function, 'negative_slope', 0.1015625)) + if name == "leaky_relu": + slope = float(getattr(layer.activation_function, "negative_slope", 0.1015625)) return lambda x: np.where(x < 0, x * slope, x) # type: ignore - raise PQuantAlkaidError(f'Unsupported PQuant activation for Alkaid conversion: {name!r}') + raise PQuantAlkaidError(f"Unsupported PQuant activation for Alkaid conversion: {name!r}") def _table_fn(table): @@ -68,7 +68,7 @@ def _table_fn(table): def apply_fn(v: np.ndarray) -> np.ndarray: with torch.no_grad(): - t = torch.as_tensor(v, dtype=torch.float32, device='cpu') + t = torch.as_tensor(v, dtype=torch.float32, device="cpu") return fn(t).detach().cpu().numpy().astype(np.float64) return apply_fn @@ -86,12 +86,12 @@ class ReplayPQuantDense(ReplayModuleBase): def call(self, input: FVArray) -> FVArray: layer = self.module - input = _replay_quantizer_if_enabled(layer, 'input_quantizer', input, 'quantize_input') + input = _replay_quantizer_if_enabled(layer, "input_quantizer", input, "quantize_input") out = input @ _final_weight(layer).T bias = _final_bias(layer) if bias.shape != (): out = out + bias - return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + return _replay_quantizer_if_enabled(layer, "output_quantizer", out, "quantize_output") class ReplayPQuantConv(ReplayModuleBase): @@ -99,7 +99,7 @@ class ReplayPQuantConv(ReplayModuleBase): def call(self, input: FVArray) -> FVArray: layer = self.module - input = _replay_quantizer_if_enabled(layer, 'input_quantizer', input, 'quantize_input') + input = _replay_quantizer_if_enabled(layer, "input_quantizer", input, "quantize_input") out = conv_nd_replay( input, _final_weight(layer), @@ -109,7 +109,7 @@ def call(self, input: FVArray) -> FVArray: dilation=layer.dilation, groups=layer.groups, ) - return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + return _replay_quantizer_if_enabled(layer, "output_quantizer", out, "quantize_output") class ReplayPQuantBatchNorm(ReplayBatchNorm): @@ -128,7 +128,7 @@ def fused_scale_offset(self) -> tuple[np.ndarray, np.ndarray]: def call(self, input: FVArray) -> FVArray: layer = self.module - input = _replay_quantizer_if_enabled(layer, 'input_quantizer', input, 'quantize_input') + input = _replay_quantizer_if_enabled(layer, "input_quantizer", input, "quantize_input") return super().call(input) @@ -137,7 +137,7 @@ class ReplayPQuantAvgPool(ReplayModuleBase): def call(self, input: FVArray) -> FVArray: layer = self.module - input = _replay_quantizer_if_enabled(layer, 'input_quantizer', input, 'quantize_input') + input = _replay_quantizer_if_enabled(layer, "input_quantizer", input, "quantize_input") out = replay_avg_pool( input, layer.kernel_size, @@ -146,7 +146,7 @@ def call(self, input: FVArray) -> FVArray: layer.ceil_mode, layer.count_include_pad, ) - return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + return _replay_quantizer_if_enabled(layer, "output_quantizer", out, "quantize_output") class ReplayPQuantActivation(ReplayModuleBase): @@ -155,9 +155,9 @@ class ReplayPQuantActivation(ReplayModuleBase): def call(self, input: FVArray) -> FVArray: layer = self.module input = _scale_by_relu_multiplier(layer, input) - input = _replay_quantizer_if_enabled(layer, 'input_quantizer', input, 'quantize_input') + input = _replay_quantizer_if_enabled(layer, "input_quantizer", input, "quantize_input") out = _activation_numpy_fn(layer)(input) - return _replay_quantizer_if_enabled(layer, 'output_quantizer', out, 'quantize_output') + return _replay_quantizer_if_enabled(layer, "output_quantizer", out, "quantize_output") class ReplayPQuantSoftmax(ReplayModuleBase): @@ -168,9 +168,9 @@ class ReplayPQuantSoftmax(ReplayModuleBase): def call(self, inputs: FVArray, mask=None) -> FVArray: module = self.module if mask is not None: - raise PQuantAlkaidError('PQSoftmax masks are not supported in Alkaid conversion.') + raise PQuantAlkaidError("PQSoftmax masks are not supported in Alkaid conversion.") if not module.built: - raise PQuantAlkaidError('PQSoftmax must be built (one real forward) before Alkaid conversion.') + raise PQuantAlkaidError("PQSoftmax must be built (one real forward) before Alkaid conversion.") return _replay_softmax(module, inputs, _table_fn) @@ -179,7 +179,7 @@ def _patch_root_quantizer_trace() -> None: import alkaid.converter.builtin.torch.main as torch_main tracer_cls = torch_main.TorchALIRTracer - marker = '__alkaid_pquant_patched_root_quantizer__' + marker = "__alkaid_pquant_patched_root_quantizer__" if getattr(tracer_cls, marker, False): return original = tracer_cls.apply_model @@ -191,7 +191,7 @@ def wrapped(self, verbose: bool, inputs: tuple[FVArray, ...]): inputs = (inputs,) replay = ReplayPQuantQuantizer(self.model) dump = replay(*inputs) - return {'inputs': tuple(inputs), 'quantizer/final': dump['final'], 'final': dump['final']}, ['final'] + return {"inputs": tuple(inputs), "quantizer/final": dump["final"], "final": dump["final"]}, ["final"] return original(self, verbose, inputs) tracer_cls.apply_model = wrapped @@ -249,11 +249,11 @@ def _register_functional_helpers() -> None: _functional_map.setdefault(torch.full, _replay_full) _functional_map.setdefault(torch.zeros_like, _replay_zeros_like) _functional_map.setdefault(torch.ones_like, _replay_ones_like) - _method_map.setdefault('pow', lambda receiver, exponent, **_kwargs: receiver**exponent) + _method_map.setdefault("pow", lambda receiver, exponent, **_kwargs: receiver**exponent) def register() -> None: """Entry point for Alkaid's ``alkaid_torch`` second-level plugin group.""" _patch_root_quantizer_trace() _register_functional_helpers() - _mark_plugin_loaded('torch') + _mark_plugin_loaded("torch") diff --git a/src/pquant/core/constants.py b/src/pquant/core/constants.py index af06610..830d062 100644 --- a/src/pquant/core/constants.py +++ b/src/pquant/core/constants.py @@ -47,7 +47,7 @@ class QuantizationGranularity(str, Enum): DB_STORAGE = "sqlite:///optuna_study.db" TORCH_BACKEND = "torch" -TF_BACKEND = 'tensorflow' +TF_BACKEND = "tensorflow" FINETUNING_DIRECTION = {"maximize", "minimize"} CONFIG_FILE = "config.yaml" diff --git a/src/pquant/core/hyperparameter_optimization.py b/src/pquant/core/hyperparameter_optimization.py index c9d4289..b7c9d36 100644 --- a/src/pquant/core/hyperparameter_optimization.py +++ b/src/pquant/core/hyperparameter_optimization.py @@ -2,7 +2,8 @@ import json import logging import os -from typing import Annotated, Callable, Dict, Optional, Union +from collections.abc import Callable +from typing import Annotated import keras import optuna @@ -56,7 +57,7 @@ class MetricFunction(BaseModel): function_name: Callable direction: str - @field_validator('direction') + @field_validator("direction") def validate_direction(cls, direction): if direction not in constants.FINETUNING_DIRECTION: raise ValueError("Direction must be 'maximize' or 'minimize'") @@ -66,16 +67,16 @@ def validate_direction(cls, direction): class PQConfig(BaseModel): hpo_parameters: BaseHyperparameterOptimizationModel pruning_parameters: Annotated[ - Union[ - CSPruningModel, - DSTPruningModel, - FITCompressPruningModel, - PDPPruningModel, - WandaPruningModel, - AutoSparsePruningModel, - ActivationPruningModel, - MDMMPruningModel, - ], + ( + CSPruningModel + | DSTPruningModel + | FITCompressPruningModel + | PDPPruningModel + | WandaPruningModel + | AutoSparsePruningModel + | ActivationPruningModel + | MDMMPruningModel + ), Field(discriminator="pruning_method"), ] quantization_parameters: BaseQuantizationModel @@ -84,10 +85,10 @@ class PQConfig(BaseModel): @classmethod def load_from_file(cls, path_to_config_file): - if path_to_config_file.endswith(('.yaml', '.yml')): + if path_to_config_file.endswith((".yaml", ".yml")): with open(path_to_config_file) as f: config_data = yaml.safe_load(f) - elif path_to_config_file.endswith('.json'): + elif path_to_config_file.endswith(".json"): with open(path_to_config_file) as f: config_data = json.load(f) else: @@ -169,11 +170,11 @@ class TuningTask: def __init__(self, config: PQConfig): self.config = config self.hyperparameters = {} - self.objectives: Dict[str, MetricFunction] = {} - self._training_function: Optional[Callable] = None - self._validation_function: Optional[Callable] = None - self._optimizer_function: Optional[Callable] = None - self._scheduler_function: Optional[Callable] = None + self.objectives: dict[str, MetricFunction] = {} + self._training_function: Callable | None = None + self._validation_function: Callable | None = None + self._optimizer_function: Callable | None = None + self._scheduler_function: Callable | None = None self.enable_mlflow = False self.tracking_uri = None self.storage_db = None @@ -293,7 +294,7 @@ def register_hyperparameter(self, name, optuna_func, *args, **kwargs): def objective(self, trial, model, train_func, valid_func, **kwargs): from pquant import add_compression_layers, train_model - + config_copy = copy.deepcopy(self.config) applied_parameters = {} for param_name, (optuna_func, func_args, func_kwargs) in self.hyperparameters.items(): @@ -315,15 +316,15 @@ def objective(self, trial, model, train_func, valid_func, **kwargs): if not applied: logging.error(f"'{param_name}' not found in config: value not applied.") - trainloader = kwargs['trainloader'] + trainloader = kwargs["trainloader"] raw_input_batch = next(iter(trainloader)) - + sample_input = raw_input_batch[0] model_copy = self.adapter.clone_model(model) model_copy = self.adapter.move_to_device(model_copy) sample_output = self.adapter.forward(model_copy, sample_input) input_shape = sample_input.shape - + compressed_model = add_compression_layers(model_copy, config_copy, input_shape) optimizer_func = self.get_optimizer_function() optimizer = optimizer_func(config_copy, compressed_model) diff --git a/src/pquant/core/keras/activations.py b/src/pquant/core/keras/activations.py index 39a7699..feb307d 100644 --- a/src/pquant/core/keras/activations.py +++ b/src/pquant/core/keras/activations.py @@ -1,7 +1,5 @@ from math import prod -from typing import Tuple from typing import TypeVar as T -from typing import Union import keras from keras import ops @@ -32,8 +30,8 @@ def __init__( self, config, activation="relu", - in_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, quantize_input=True, quantize_output=False, enable_ebops=True, @@ -226,18 +224,18 @@ class PQSoftmax(keras.layers.Layer): def __init__( self, config, - axis: Union[int, Tuple[int, ...]] = -1, + axis: int | tuple[int, ...] = -1, stable: bool = True, input_scaler: float = 1.0, parallelization_factor: int = -1, quantize_input: bool = True, quantize_output: bool = False, - in_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, - exp_in_quant_bits: Tuple[T, T, T] = None, - exp_out_quant_bits: Tuple[T, T, T] = None, - inv_in_quant_bits: Tuple[T, T, T] = None, - inv_out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, + exp_in_quant_bits: tuple[T, T, T] = None, + exp_out_quant_bits: tuple[T, T, T] = None, + inv_in_quant_bits: tuple[T, T, T] = None, + inv_out_quant_bits: tuple[T, T, T] = None, **kwargs, ): super().__init__(**kwargs) diff --git a/src/pquant/core/keras/convert_to_onnx.py b/src/pquant/core/keras/convert_to_onnx.py index 35263c2..781b8fa 100644 --- a/src/pquant/core/keras/convert_to_onnx.py +++ b/src/pquant/core/keras/convert_to_onnx.py @@ -114,9 +114,7 @@ def _quant_node(name_prefix, input_name, rounding_mode, k, i, f, initializers, o # --------------------------------------------------------------------------- -def _qdq_node( - name_prefix, input_name, rounding_mode, k, i, f, initializers, overflow_mode="SAT", include_clip=True -): # noqa: ARG001 +def _qdq_node(name_prefix, input_name, rounding_mode, k, i, f, initializers, overflow_mode="SAT", include_clip=True): """Build QuantizeLinear+DequantizeLinear nodes, optionally preceded by a Clip. Returns ([nodes], output_name). Set include_clip=False to skip the Clip node diff --git a/src/pquant/core/keras/layers.py b/src/pquant/core/keras/layers.py index dc996e2..c607c79 100644 --- a/src/pquant/core/keras/layers.py +++ b/src/pquant/core/keras/layers.py @@ -1,5 +1,5 @@ from math import prod -from typing import Tuple, TypeVar +from typing import TypeVar import keras from keras import constraints, initializers, ops, regularizers @@ -63,10 +63,10 @@ def __init__( layer_type, quantize_input=True, quantize_output=False, - in_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + weight_quant_bits: tuple[T, T, T] = None, + bias_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, weight_quant_granularity=None, in_quant_granularity=None, bias_quant_granularity=None, @@ -471,10 +471,10 @@ def __init__( bias: bool = True, device=None, dtype=None, - in_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + weight_quant_bits: tuple[T, T, T] = None, + bias_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, weight_quant_granularity=None, in_quant_granularity=None, bias_quant_granularity=None, @@ -609,10 +609,10 @@ def __init__( activity_regularizer=None, kernel_constraint=None, bias_constraint=None, - in_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + weight_quant_bits: tuple[T, T, T] = None, + bias_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, weight_quant_granularity=None, in_quant_granularity=None, bias_quant_granularity=None, @@ -889,10 +889,10 @@ def __init__( kernel_size, quantize_input=True, quantize_output=False, - in_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + weight_quant_bits: tuple[T, T, T] = None, + bias_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, weight_quant_granularity=None, in_quant_granularity=None, bias_quant_granularity=None, @@ -1049,10 +1049,10 @@ def __init__( units, quantize_input=True, quantize_output=False, - in_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + weight_quant_bits: tuple[T, T, T] = None, + bias_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, weight_quant_granularity=None, in_quant_granularity=None, bias_quant_granularity=None, @@ -1405,8 +1405,8 @@ def __init__( config, quantize_input=True, quantize_output=False, - in_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, in_quant_granularity=None, out_quant_granularity=None, **kwargs, @@ -1548,8 +1548,8 @@ def __init__( pool_size, quantize_input=True, quantize_output=False, - in_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, in_quant_granularity=None, out_quant_granularity=None, strides=None, @@ -1594,8 +1594,8 @@ def __init__( pool_size, quantize_input=True, quantize_output=False, - in_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, in_quant_granularity=None, out_quant_granularity=None, strides=None, @@ -1682,11 +1682,11 @@ def __init__( quantize_input: bool = True, quantize_output: bool = False, approximate_softmax: bool = False, - in_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, - attn_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + weight_quant_bits: tuple[T, T, T] = None, + bias_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, + attn_quant_bits: tuple[T, T, T] = None, in_quant_granularity=None, out_quant_granularity=None, param_quant_granularity=None, diff --git a/src/pquant/core/keras/pruning_methods/constraint_functions.py b/src/pquant/core/keras/pruning_methods/constraint_functions.py index 431cdd2..f835346 100644 --- a/src/pquant/core/keras/pruning_methods/constraint_functions.py +++ b/src/pquant/core/keras/pruning_methods/constraint_functions.py @@ -24,19 +24,19 @@ def __init__(self, lmbda_init=1.0, scale=1.0, damping=1.0, **kwargs): super().__init__(**kwargs) self.scale = self.add_weight( - name='scale', + name="scale", shape=(), initializer=lambda shape, dtype: ops.convert_to_tensor(scale, dtype=dtype), trainable=False, ) self.damping = self.add_weight( - name='damping', + name="damping", shape=(), initializer=lambda shape, dtype: ops.convert_to_tensor(damping, dtype=dtype), trainable=False, ) self.lmbda = self.add_weight( - name=f'{self.name}_lmbda', + name=f"{self.name}_lmbda", shape=(), initializer=lambda shape, dtype: ops.convert_to_tensor(lmbda_init, dtype=dtype), trainable=self.use_grad_, @@ -44,7 +44,7 @@ def __init__(self, lmbda_init=1.0, scale=1.0, damping=1.0, **kwargs): if not self.use_grad_: self.prev_infs = self.add_weight( - name=f'{self.name}_prev_infs', + name=f"{self.name}_prev_infs", shape=(), initializer=lambda shape, dtype: ops.convert_to_tensor(0.0, dtype=dtype), trainable=False, diff --git a/src/pquant/core/keras/pruning_methods/metric_functions.py b/src/pquant/core/keras/pruning_methods/metric_functions.py index 0f22b5e..071b84b 100644 --- a/src/pquant/core/keras/pruning_methods/metric_functions.py +++ b/src/pquant/core/keras/pruning_methods/metric_functions.py @@ -6,10 +6,10 @@ class UnstructuredSparsityMetric: """Calculates the ratio of non-zero weights in a tensor.""" - def __init__(self, l0_mode='coarse', scale_mode="mean", epsilon=1e-3, target_sparsity=0.8, alpha=100.0): + def __init__(self, l0_mode="coarse", scale_mode="mean", epsilon=1e-3, target_sparsity=0.8, alpha=100.0): # Note: scale_mode:"sum" give very high losses for large model - assert l0_mode in ['coarse', 'smooth'], "Mode must be 'coarse' or 'smooth'" - assert scale_mode in ['sum', 'mean'], "Scale mode must be 'sum' or 'mean'" + assert l0_mode in ["coarse", "smooth"], "Mode must be 'coarse' or 'smooth'" + assert scale_mode in ["sum", "mean"], "Scale mode must be 'sum' or 'mean'" assert 0 <= target_sparsity <= 1, "target_sparsity must be between 0 and 1" self.l0_mode = l0_mode self.scale_mode = scale_mode @@ -24,14 +24,14 @@ def __init__(self, l0_mode='coarse', scale_mode="mean", epsilon=1e-3, target_spa def build(self): # l0 term -> number of zero weights/number of weights - if self.l0_mode == 'coarse': + if self.l0_mode == "coarse": self.l0_fn = self._coarse_l0 - elif self.l0_mode == 'smooth': + elif self.l0_mode == "smooth": self.l0_fn = self._smooth_l0 - if self.scale_mode == 'mean': + if self.scale_mode == "mean": self._scaling = self._mean_scaling - elif self.scale_mode == 'sum': + elif self.scale_mode == "sum": self._scaling = self._sum_scaling def _sum_scaling(self, fn_value, num): diff --git a/src/pquant/core/torch/activations.py b/src/pquant/core/torch/activations.py index 4aa1773..d7e356d 100644 --- a/src/pquant/core/torch/activations.py +++ b/src/pquant/core/torch/activations.py @@ -1,5 +1,5 @@ from math import prod -from typing import Tuple, TypeVar, Union +from typing import TypeVar import torch import torch.nn as nn @@ -37,8 +37,8 @@ def __init__( self, config, activation="relu", - in_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, quantize_input=True, quantize_output=False, enable_ebops=True, @@ -224,18 +224,18 @@ class PQSoftmax(nn.Module): def __init__( self, config, - axis: Union[int, Tuple[int, ...]] = -1, + axis: int | tuple[int, ...] = -1, stable: bool = True, input_scaler: float = 1.0, parallelization_factor: int = -1, quantize_input: bool = True, quantize_output: bool = False, - in_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, - exp_in_quant_bits: Tuple[T, T, T] = None, - exp_out_quant_bits: Tuple[T, T, T] = None, - inv_in_quant_bits: Tuple[T, T, T] = None, - inv_out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, + exp_in_quant_bits: tuple[T, T, T] = None, + exp_out_quant_bits: tuple[T, T, T] = None, + inv_in_quant_bits: tuple[T, T, T] = None, + inv_out_quant_bits: tuple[T, T, T] = None, **kwargs, ): super().__init__(**kwargs) diff --git a/src/pquant/core/torch/convert_to_onnx.py b/src/pquant/core/torch/convert_to_onnx.py index a61aeaa..1601b73 100644 --- a/src/pquant/core/torch/convert_to_onnx.py +++ b/src/pquant/core/torch/convert_to_onnx.py @@ -114,9 +114,7 @@ def _quant_node(name_prefix, input_name, rounding_mode, k, i, f, initializers, o # --------------------------------------------------------------------------- -def _qdq_node( - name_prefix, input_name, rounding_mode, k, i, f, initializers, overflow_mode="SAT", include_clip=True -): # noqa: ARG001 +def _qdq_node(name_prefix, input_name, rounding_mode, k, i, f, initializers, overflow_mode="SAT", include_clip=True): """Build QuantizeLinear+DequantizeLinear nodes, optionally preceded by a Clip. Returns ([nodes], output_name). Set include_clip=False to skip the Clip node @@ -1388,7 +1386,7 @@ def _check_q_int16(arr: np.ndarray, frac_bits: int, name: str) -> None: f"(max abs round error = {np.max(np.abs(scaled - rounded)):.6g})" ) if rounded.min() < INT16_MIN or rounded.max() > INT16_MAX: - raise ValueError(f"{name} overflows int16 at Q{frac_bits} " f"(range [{rounded.min()}, {rounded.max()}])") + raise ValueError(f"{name} overflows int16 at Q{frac_bits} (range [{rounded.min()}, {rounded.max()}])") _check_q_int16(gamma, GAMMA_F, "gamma") _check_q_int16(beta, BETA_F, "beta") @@ -1638,7 +1636,7 @@ def _resolve_perm_dims(args, rank: int) -> list[int]: container = node_to_name[node.args[0]] if not isinstance(container, tuple): raise TypeError( - f"operator.getitem on non-tuple node {node.args[0].name!r} " f"is not supported in FX ONNX export" + f"operator.getitem on non-tuple node {node.args[0].name!r} is not supported in FX ONNX export" ) node_to_name[node] = container[node.args[1]] continue diff --git a/src/pquant/core/torch/distillers.py b/src/pquant/core/torch/distillers.py index fce5fe0..dcb11ca 100644 --- a/src/pquant/core/torch/distillers.py +++ b/src/pquant/core/torch/distillers.py @@ -2,7 +2,7 @@ import os import tempfile -from typing import Callable, Iterable +from collections.abc import Callable, Iterable import torch import torch.nn as nn @@ -114,10 +114,10 @@ def precompute_layer_io( def teacher_post(m: nn.Module, inp: tuple, out: torch.Tensor) -> None: teacher_output = out[0] if isinstance(out, tuple) else out - teacher_captured_data['out'] = teacher_output.detach().cpu() + teacher_captured_data["out"] = teacher_output.detach().cpu() def teacher_pre(m: nn.Module, inp: tuple) -> None: - teacher_captured_data['inp'] = inp[0].detach().cpu() + teacher_captured_data["inp"] = inp[0].detach().cpu() teacher_output_hook = teacher_layer.register_forward_hook(teacher_post) teacher_input_hook = teacher_layer.register_forward_pre_hook(teacher_pre) @@ -130,7 +130,7 @@ def teacher_pre(m: nn.Module, inp: tuple) -> None: x = x.to(self.device) self.teacher(x) torch.save( - (teacher_captured_data['inp'], teacher_captured_data['out']), + (teacher_captured_data["inp"], teacher_captured_data["out"]), os.path.join(cache_dir, f"{n_batches:08d}.pt"), ) n_batches += 1 @@ -162,10 +162,10 @@ def val_layer_loss( s_captured: dict[str, torch.Tensor] = {} def teacher_post(m: nn.Module, inp: tuple, out) -> None: - t_captured['out'] = (out[0] if isinstance(out, tuple) else out).detach() + t_captured["out"] = (out[0] if isinstance(out, tuple) else out).detach() def student_hook(m: nn.Module, inp: tuple, out) -> None: - s_captured['out'] = out[0] if isinstance(out, tuple) else out + s_captured["out"] = out[0] if isinstance(out, tuple) else out h_t = teacher_layer.register_forward_hook(teacher_post) h_s = student_layer.register_forward_hook(student_hook) @@ -179,7 +179,7 @@ def student_hook(m: nn.Module, inp: tuple, out) -> None: x = x.to(self.device) self.teacher(x) self.student(x) - batch_losses.append(self.loss_fn(s_captured['out'], t_captured['out']).item()) + batch_losses.append(self.loss_fn(s_captured["out"], t_captured["out"]).item()) finally: h_t.remove() h_s.remove() @@ -265,7 +265,7 @@ def student_hook(m: nn.Module, inp: tuple, out) -> None: hooks = [h_t, h_s] for epoch in range(n_epochs): - if getattr(student_layer, 'enable_pruning', False): + if getattr(student_layer, "enable_pruning", False): student_layer.pruning_layer.pre_epoch_function(epoch, n_epochs) batch_losses: list[float] = [] @@ -296,7 +296,7 @@ def student_hook(m: nn.Module, inp: tuple, out) -> None: mean_loss = sum(batch_losses) / len(batch_losses) epoch_losses.append(mean_loss) - if getattr(student_layer, 'enable_pruning', False): + if getattr(student_layer, "enable_pruning", False): student_layer.pruning_layer.post_epoch_function(epoch, n_epochs) val_loss: float | None = None if val_dataloader is not None: diff --git a/src/pquant/core/torch/fit_compress.py b/src/pquant/core/torch/fit_compress.py index 762863c..fb67528 100644 --- a/src/pquant/core/torch/fit_compress.py +++ b/src/pquant/core/torch/fit_compress.py @@ -184,7 +184,6 @@ def print_info_bits(model): class node: - def __init__( self, matrices_params_layerwise, @@ -232,7 +231,7 @@ def __init__( self.unquantized_weights = unquantized_weights self.int_bits = int_bits self.frac_bits = frac_bits - self.key = ''.join(random.choices(string.ascii_uppercase + string.digits, k=20)) + self.key = "".join(random.choices(string.ascii_uppercase + string.digits, k=20)) def extract_config_from_node(self, layer_names): """ @@ -250,13 +249,12 @@ def extract_config_from_node(self, layer_names): layer_name: [i_bits, f_bits] for layer_name, i_bits, f_bits in zip(layer_names, self.int_bits, self.frac_bits) } - config = {'quant_config': quant_config, 'pruning_metrics': self.pruning_metrics} + config = {"quant_config": quant_config, "pruning_metrics": self.pruning_metrics} return config class FITcompress: - def __init__(self, model, device, dataloader, criterion, config, layerwise_pruning=False, input_shape=None): """ Calculate initial EF of the uncompressed model and set up quantization & @@ -320,7 +318,7 @@ def __init__(self, model, device, dataloader, criterion, config, layerwise_pruni ) # for N:M pruning in Wanda, use 50% pruning cap during FITcompress - if self.config.pruning_parameters.pruning_method == 'wanda' and type(self.config.pruning_parameters.N) is int: + if self.config.pruning_parameters.pruning_method == "wanda" and type(self.config.pruning_parameters.N) is int: self.pruning_schedule = 0.5 * ( 1 - np.logspace( @@ -333,15 +331,15 @@ def __init__(self, model, device, dataloader, criterion, config, layerwise_pruni # Dictionary structure allows us to possibly iterate over multiple different pruning metrics # but currently only one as in FITcompress, the target pruning sparsity, i.e. percentage - pruning_metrics = {'percentage': 0} + pruning_metrics = {"percentage": 0} # If we want to find sparsity targets per layer (not part of FITcompress paper) if layerwise_pruning: self.pruning_schedulers_layerwise = self.get_pruning_schedulers_layer_specific( - matrices_params_layerwise, None, mode='fit' + matrices_params_layerwise, None, mode="fit" ) # Add the layer-specific starting pruning percentages to the current metric - pruning_metrics = pruning_metrics | {f'{self.layer_names[i]}_percentage': 0 for i in range(self.n_layers)} + pruning_metrics = pruning_metrics | {f"{self.layer_names[i]}_percentage": 0 for i in range(self.n_layers)} # Initialize the first node in the compression space self.initial_node = node( @@ -368,7 +366,7 @@ def __init__(self, model, device, dataloader, criterion, config, layerwise_pruni # Intialize a list to store nodes that can be traversed during the path finding process self.potential_nodes = [self.initial_node] - def get_pruning_schedulers_layer_specific(self, matrices_params_layerwise, global_sparsity_scheduler, mode='fit'): + def get_pruning_schedulers_layer_specific(self, matrices_params_layerwise, global_sparsity_scheduler, mode="fit"): """ Calculates layer-specific pruning schedulers. The idea is that layers with weights that are not that much affected by pertubation should be pruned more/faster than layers with weights @@ -383,7 +381,7 @@ def get_pruning_schedulers_layer_specific(self, matrices_params_layerwise, globa """ schedulers = {} - if mode == 'fit': + if mode == "fit": # Get the layer-wise FIT scores of the initial model _, FIT_layerwise = self.fit_computer.get_FIT_old( FeM=self.FeM, params_after=matrices_params_layerwise, same_theta=True @@ -396,7 +394,6 @@ def get_pruning_schedulers_layer_specific(self, matrices_params_layerwise, globa max_importance = max(FIT_layerwise_summed) for layer_idx, importance in enumerate(FIT_layerwise_summed): - # Scale importance between 0 and 1 importance_ratio = (importance - min_importance) / (max_importance - min_importance) @@ -452,7 +449,7 @@ def assign_parameters(self, model, params): for _, module in model.named_modules(): if isinstance(module, (PQDense, PQConv2d)): for name_param, matrix_param in list(module.named_parameters()): - if name_param.endswith('_weight'): + if name_param.endswith("_weight"): matrix_param.data = nn.parameter.Parameter(params[i].to(self.device)) matrix_param.collect = True i += 1 @@ -643,7 +640,6 @@ def add_pruning_layer_specific(self, current_node, pruning_metrics, layer_idx=No current_node_matrices_params_layerwise = [] # Now iterate through all layers for idx, curr_pruning_percentage in enumerate(pruning_metrics.values()): - if idx == 0: # Global pruning percentage continue @@ -768,7 +764,7 @@ def astar(self, config): """ iterations = 0 while len(self.potential_nodes) > 0 and iterations < 1000: - logging.info(f'Iteration : {iterations} ') + logging.info(f"Iteration : {iterations} ") next_best_node = None @@ -777,7 +773,6 @@ def astar(self, config): for p_node in self.potential_nodes: # If we find a node with wanted compression rate, we can return it and stop the A* algorithm if p_node.curr_compression_rate < self.compression_goal: - logging.info( f"Optimal node found with full distance {p_node.full_dist}, " f"compression rate {p_node.curr_compression_rate}, " @@ -862,7 +857,6 @@ def create_neighbours(self, current_node): """ if self.config.fitcompress_parameters.approximate: - # Update FeM for the best node and use it when creating the neighbours for quantization. # This leads to num_layers less FIT calculations, as we do not need to calculate the FeM again, # which reduces runtime @@ -885,7 +879,6 @@ def create_neighbours(self, current_node): logging.info("Current node states for quantization & pruning: ", current_node_state) if self.config.fitcompress_parameters.optimize_quantization: for layer_idx in range(self.n_layers): - # Set neighbour state to current state neighbour_node_state = current_node_state.copy() @@ -930,7 +923,6 @@ def create_neighbours(self, current_node): self.potential_nodes.append(neighbour_node) if self.config.fitcompress_parameters.optimize_pruning: - # Set neighbour state to current state neighbour_node_state = current_node_state.copy() @@ -969,7 +961,6 @@ def create_neighbours(self, current_node): current_node=current_node, pruning_metrics=neighbour_node_pruning_metrics ) else: - neighbour_node_parameters_layerwise, neighbour_node_unquantized_parameters_layerwise = self.add_pruning( current_node=current_node, params=current_node.parameters.copy(), @@ -1165,7 +1156,6 @@ def calculate_current_compression_rate(self, params_layerwise, quant_config): class FIT: - def __init__(self, model, device, input_spec): """ Initialize the FIT class, which is used to compute the FIT values for quantization and pruning. @@ -1213,12 +1203,11 @@ def get_model_weights(self, model): layer_names = [] # Iterate through all modules in the model for name, module in model.named_modules(): - if isinstance(module, (PQDense, PQConv2d)): layer_names.append(name) for name_param, matrix_param in list(module.named_parameters()): # Search for the weights - if name_param.endswith('_weight'): + if name_param.endswith("_weight"): matrices_params_layerwise.append(matrix_param) # Set their collect flag to True (later on we can then access them easily like this) matrix_param.collect = True @@ -1272,7 +1261,7 @@ def hook_removal(self): self.hooks.clear() assert len(self.hooks) == 0, "Hooks were not removed properly!" - def get_loss(self, model, data_batch, target_batch, loss_func, mode='mini-batch'): + def get_loss(self, model, data_batch, target_batch, loss_func, mode="mini-batch"): """ This function triggers the loss calcuation of a model. We use it such that we can then calculate gradients which are @@ -1298,16 +1287,16 @@ def get_loss(self, model, data_batch, target_batch, loss_func, mode='mini-batch' output = model(data_batch) - if mode == 'mini-batch': + if mode == "mini-batch": # Check which loss_func instance is active if isinstance(loss_func, torch.nn.CrossEntropyLoss): # Calculate loss based on mini-batch and averaged over it loss_func = torch.nn.CrossEntropyLoss() - if mode == 'sample': + if mode == "sample": if isinstance(loss_func, torch.nn.CrossEntropyLoss): # Calculate loss for each sample - loss_func = torch.nn.CrossEntropyLoss(reduce=False, reduction='none') + loss_func = torch.nn.CrossEntropyLoss(reduce=False, reduction="none") loss = loss_func(output, target_batch) @@ -1401,7 +1390,7 @@ def get_EF(self, model, data_loader, loss_func, tolerance=1e-3, min_iterations=1 if data_batch.size(0) != batch_size: continue # Uneven batches break loop - loss = self.get_loss(model, data_batch, target_batch, loss_func, mode='mini-batch') + loss = self.get_loss(model, data_batch, target_batch, loss_func, mode="mini-batch") curr_batch_matrices_params_layerwise = [] curr_batch_minmax_range_params_layerwise = [] for weights in model.parameters(): @@ -1635,7 +1624,6 @@ def get_FIT_real_values(self, params_before, EF_trace_params_layerwise, params_a curr_FIT += EF_trace * delta_theta else: - for theta, EF_trace in zip(params_before, EF_trace_params_layerwise): # Calculate the squared difference between the parameters before and after delta_theta = torch.sum(theta.detach().cpu() ** 2) @@ -1664,10 +1652,8 @@ def get_FIT_old(self, params_before=None, FeM=None, params_after=None, same_thet curr_FIT = 0 if not same_theta: - # Taken from compute_fake_FIT_params() for theta_before, theta_after, layer_FeM in zip(params_before, params_after, FeM): - curr_FIT_layer = torch.sum( layer_FeM * (theta_before.detach().cpu() - theta_after.detach().cpu()) ** 2 ).numpy() @@ -1680,7 +1666,6 @@ def get_FIT_old(self, params_before=None, FeM=None, params_after=None, same_thet # Taken from generate_FIT_pruning_importance() for theta_after, layer_FeM in zip(params_after, FeM): - curr_FIT_layer = layer_FeM * (theta_after.detach().cpu() ** 2) FIT_layerwise.append(curr_FIT_layer) diff --git a/src/pquant/core/torch/fixed_point_quantizer.py b/src/pquant/core/torch/fixed_point_quantizer.py index 7ed3bd5..b4f9bc5 100644 --- a/src/pquant/core/torch/fixed_point_quantizer.py +++ b/src/pquant/core/torch/fixed_point_quantizer.py @@ -10,7 +10,7 @@ round_mode_registry: dict[str, Callable[[Any], Any]] = {} saturation_mode_registry: dict[str, Callable[[Any, Any, Any, Any], Any]] = {} -T = TypeVar('T', bound=ArrayLike) +T = TypeVar("T", bound=ArrayLike) def _clip(x, min_value, max_value): @@ -34,44 +34,44 @@ def wrapper(x): return inner -@rnd_mode('TRN') +@rnd_mode("TRN") def floor(x): return torch.floor(x) -@rnd_mode('RND') +@rnd_mode("RND") def round(x): # Round to nearest, ties positive infinity. return torch.floor(x + 0.5) -@rnd_mode('RND_CONV') +@rnd_mode("RND_CONV") def round_conv(x): # Round to nearest, ties to even. return torch.round(x) -@rnd_mode('TRN_ZERO') +@rnd_mode("TRN_ZERO") def floor_zero(x): # Truncate towards zero. sign = torch.sign(x) return torch.floor(torch.abs(x)) * sign # type: ignore -@rnd_mode('RND_ZERO') +@rnd_mode("RND_ZERO") def round_zero(x): # Round to nearest, ties towards zero. sign = torch.sign(x) return -torch.floor(-torch.abs(x) + 0.5) * sign # type:ignore -@rnd_mode('RND_MIN_INF') +@rnd_mode("RND_MIN_INF") def round_min_inf(x): # Round to nearest, ties towards negative infinity. return -torch.floor(-x + 0.5) # type:ignore -@rnd_mode('RND_INF') +@rnd_mode("RND_INF") def round_inf(x): # Round to nearest, ties away from zero. sign = torch.sign(x) @@ -91,7 +91,7 @@ def inner(func): return inner -@sat_mode('WRAP') +@sat_mode("WRAP") def wrap(x, k, i, f): xs = x bk = i + k @@ -99,7 +99,7 @@ def wrap(x, k, i, f): return (xs + bias) % (2.0**bk) - bias -@sat_mode('SAT') +@sat_mode("SAT") def sat(x, k, i, f): f_eps = 2.0 ** (-f) __max = 2.0**i @@ -109,7 +109,7 @@ def sat(x, k, i, f): return r -@sat_mode('SAT_SYM') +@sat_mode("SAT_SYM") def sat_sym(x, k, i, f): f_eps = 2.0 ** (-f) _max = 2.0**i - f_eps @@ -118,7 +118,7 @@ def sat_sym(x, k, i, f): return r -@sat_mode('WRAP_SM') +@sat_mode("WRAP_SM") def wrap_sm_fn(x, k, i, f, training=None, quant_fn: Callable = lambda x: x): # x=ops.round(x*2.**f) # High and low bounds are reflective. When overflows, can be less trash than WARP but still more trash than SAT. @@ -149,20 +149,20 @@ def round(self, x, f: Any = 1.0): def saturate(self, x, k, i, f): return self.sat_fn(x, k, i, f) - def __init__(self, round_mode: str = 'TRN', overflow_mode: str = 'WRAP'): + def __init__(self, round_mode: str = "TRN", overflow_mode: str = "WRAP"): round_mode = round_mode.upper() overflow_mode = overflow_mode.upper() self.stochastic = False - if round_mode.startswith('S_'): + if round_mode.startswith("S_"): round_mode = round_mode[2:] self.stochastic = True - if overflow_mode == 'WRAP_SM': + if overflow_mode == "WRAP_SM": assert round_mode in ( - 'RND', - 'RND_CONV', - ), 'WRAP_SM only supports RND and RND_CONV rounding modes in this implementation.' + "RND", + "RND_CONV", + ), "WRAP_SM only supports RND and RND_CONV rounding modes in this implementation." self.round_mode = round_mode self.overflow_mode = overflow_mode @@ -177,10 +177,10 @@ def forward(self, x, k, i, f, training=False): # will be clipped off anyway. Thus have saturation before rounding, except for # wrap mode, which doesn't round during training. - if self.overflow_mode != 'WRAP': + if self.overflow_mode != "WRAP": x = self.saturate(x, k, i, f) x = self.round(x, f) - if self.overflow_mode == 'WRAP' and not training: + if self.overflow_mode == "WRAP" and not training: x = self.saturate(x, k, i, f) return x @@ -194,14 +194,14 @@ def quant_fn(x): def __call__(self, x, k, i, f, training=False, seed_gen=None): i = torch.maximum(i, -f).detach() + (i - i.detach()) # type: ignore if self.stochastic and training: - assert seed_gen is not None, 'Seed generator must be provided for stochastic rounding.' - if self.overflow_mode != 'WRAP_SM': + assert seed_gen is not None, "Seed generator must be provided for stochastic rounding." + if self.overflow_mode != "WRAP_SM": return self.forward(x, k, i, f, training) else: return self.forward_wrap_sm(x, k, i, f, training) -def get_fixed_quantizer(round_mode: str = 'TRN', overflow_mode: str = 'WRAP'): +def get_fixed_quantizer(round_mode: str = "TRN", overflow_mode: str = "WRAP"): """Get a stateless fixed-point quantizer given the round and overflow mode. The quantizer is differentiable w.r.t. to the input and f, also i if using saturation overflow mode. diff --git a/src/pquant/core/torch/layers.py b/src/pquant/core/torch/layers.py index 85e302f..7badc30 100644 --- a/src/pquant/core/torch/layers.py +++ b/src/pquant/core/torch/layers.py @@ -1,6 +1,6 @@ import math import typing -from typing import Optional, Tuple, TypeVar, Union +from typing import TypeVar import torch import torch.nn as nn @@ -12,7 +12,7 @@ from pquant.core.torch.utils import get_pruning_layer if typing.TYPE_CHECKING: - from pquant.core.torch.fit_compress import call_fitcompress # noqa: 401 + pass # noqa: 401 T = TypeVar("T") @@ -49,10 +49,10 @@ def __init__( quantize_input=True, quantize_output=False, enable_pruning: bool = None, - in_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + weight_quant_bits: tuple[T, T, T] = None, + bias_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, weight_quant_granularity=None, in_quant_granularity=None, bias_quant_granularity=None, @@ -275,10 +275,10 @@ def __init__( enable_pruning: bool = None, device=None, dtype=None, - in_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + weight_quant_bits: tuple[T, T, T] = None, + bias_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, weight_quant_granularity=None, in_quant_granularity=None, bias_quant_granularity=None, @@ -447,7 +447,7 @@ def __init__( out_channels: int, kernel_size: _size_2_t, stride: _size_2_t = 1, - padding: Union[str, _size_2_t] = 0, + padding: str | _size_2_t = 0, dilation: _size_2_t = 1, groups: int = 1, bias: bool = True, @@ -457,10 +457,10 @@ def __init__( quantize_input=True, quantize_output=False, enable_pruning: bool = None, - in_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + weight_quant_bits: tuple[T, T, T] = None, + bias_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, weight_quant_granularity=None, in_quant_granularity=None, bias_quant_granularity=None, @@ -508,7 +508,7 @@ def __init__( out_channels: int, kernel_size: _size_1_t, stride: _size_1_t = 1, - padding: Union[str, _size_1_t] = 0, + padding: str | _size_1_t = 0, dilation: _size_1_t = 1, groups: int = 1, bias: bool = True, @@ -518,10 +518,10 @@ def __init__( quantize_input=True, quantize_output=False, enable_pruning: bool = None, - in_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + weight_quant_bits: tuple[T, T, T] = None, + bias_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, weight_quant_granularity=None, in_quant_granularity=None, bias_quant_granularity=None, @@ -583,8 +583,8 @@ def __init__( config, quantize_input=True, quantize_output=False, - in_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, in_quant_granularity=None, out_quant_granularity=None, **kwargs, @@ -698,8 +698,8 @@ def __init__( count_include_pad: bool = True, quantize_input=True, quantize_output=False, - in_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, in_quant_granularity=None, out_quant_granularity=None, **kwargs, @@ -736,11 +736,11 @@ def __init__( padding: _size_2_t = 0, ceil_mode: bool = False, count_include_pad: bool = True, - divisor_override: Optional[int] = None, + divisor_override: int | None = None, quantize_input=True, quantize_output=False, - in_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, in_quant_granularity=None, out_quant_granularity=None, **kwargs, @@ -927,15 +927,15 @@ def __init__( config, num_features: int, eps: float = 1e-5, - momentum: typing.Optional[float] = 0.1, + momentum: float | None = 0.1, affine: bool = True, track_running_stats: bool = True, device=None, dtype=None, quantize_input=True, - in_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + weight_quant_bits: tuple[T, T, T] = None, + bias_quant_bits: tuple[T, T, T] = None, in_quant_granularity=None, weight_quant_granularity=None, bias_quant_granularity=None, @@ -959,15 +959,15 @@ def __init__( config, num_features: int, eps: float = 1e-5, - momentum: typing.Optional[float] = 0.1, + momentum: float | None = 0.1, affine: bool = True, track_running_stats: bool = True, device=None, dtype=None, quantize_input=True, - in_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + weight_quant_bits: tuple[T, T, T] = None, + bias_quant_bits: tuple[T, T, T] = None, in_quant_granularity=None, weight_quant_granularity=None, bias_quant_granularity=None, @@ -989,7 +989,7 @@ class PQLayerNorm(nn.LayerNorm): def __init__( self, config, - normalized_shape: Union[int, Tuple[int, ...], torch.Size], + normalized_shape: int | tuple[int, ...] | torch.Size, eps: float = 1e-5, elementwise_affine: bool = True, bias: bool = True, @@ -997,10 +997,10 @@ def __init__( dtype=None, quantize_input=True, quantize_output=False, - in_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, + weight_quant_bits: tuple[T, T, T] = None, + bias_quant_bits: tuple[T, T, T] = None, in_quant_granularity=None, out_quant_granularity=None, weight_quant_granularity=None, @@ -1226,11 +1226,11 @@ def __init__( quantize_input: bool = True, quantize_output: bool = False, approximate_softmax: bool = False, - in_quant_bits: Tuple[T, T, T] = None, - weight_quant_bits: Tuple[T, T, T] = None, - bias_quant_bits: Tuple[T, T, T] = None, - out_quant_bits: Tuple[T, T, T] = None, - attn_quant_bits: Tuple[T, T, T] = None, + in_quant_bits: tuple[T, T, T] = None, + weight_quant_bits: tuple[T, T, T] = None, + bias_quant_bits: tuple[T, T, T] = None, + out_quant_bits: tuple[T, T, T] = None, + attn_quant_bits: tuple[T, T, T] = None, in_quant_granularity=None, out_quant_granularity=None, param_quant_granularity=None, @@ -1323,10 +1323,10 @@ def forward( query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, - key_padding_mask: Optional[torch.Tensor] = None, - attn_mask: Optional[torch.Tensor] = None, + key_padding_mask: torch.Tensor | None = None, + attn_mask: torch.Tensor | None = None, need_weights: bool = True, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + ) -> tuple[torch.Tensor, torch.Tensor | None]: if not self.batch_first: # (T, B, E) -> (B, T, E) query = query.transpose(0, 1) @@ -1379,8 +1379,7 @@ def forward( def extra_repr(self) -> str: return ( - f"embed_dim={self.embed_dim}, num_heads={self.num_heads}, " - f"dropout={self.dropout}, batch_first={self.batch_first}" + f"embed_dim={self.embed_dim}, num_heads={self.num_heads}, dropout={self.dropout}, batch_first={self.batch_first}" ) diff --git a/src/pquant/core/torch/optimizers.py b/src/pquant/core/torch/optimizers.py index a85022e..8914454 100644 --- a/src/pquant/core/torch/optimizers.py +++ b/src/pquant/core/torch/optimizers.py @@ -20,20 +20,20 @@ def step(self, closure=None): # Store the old params old_params = [] for group in self.param_groups: - old_params.append({param: param.data.clone() for param in group['params'] if param.grad is not None}) + old_params.append({param: param.data.clone() for param in group["params"] if param.grad is not None}) # Perform the standard AdamW step loss = super().step(closure) # Perform the pWD step for group, old_group in zip(self.param_groups, old_params): - lambda_p_group = group.get('lambda_p', self.lambda_p) # support prams groups + lambda_p_group = group.get("lambda_p", self.lambda_p) # support prams groups if lambda_p_group > 0: # Apply regularization only for lambda_p > 0 - for param in group['params']: + for param in group["params"]: if param.grad is None: continue # Use old parameters in the decay factor param_old = old_group[param] X = param_old.abs() ** (2 - self.p_norm) - update_term = X / (X + self.p_norm * group['lr'] * lambda_p_group) + update_term = X / (X + self.p_norm * group["lr"] * lambda_p_group) # pWD step param.data.mul_(update_term) return loss @@ -50,20 +50,20 @@ def step(self, closure=None): # Store the old params old_params = [] for group in self.param_groups: - old_params.append({param: param.data.clone() for param in group['params'] if param.grad is not None}) + old_params.append({param: param.data.clone() for param in group["params"] if param.grad is not None}) # Perform the standard SGD step loss = super().step(closure) # Perform the pWD step for group, old_group in zip(self.param_groups, old_params): - lambda_p_group = group.get('lambda_p', self.lambda_p) # support prams groups + lambda_p_group = group.get("lambda_p", self.lambda_p) # support prams groups if lambda_p_group > 0: # Apply regularization only for lambda_p > 0 - for param in group['params']: + for param in group["params"]: if param.grad is None: continue # Use old parameters in the decay factor param_old = old_group[param] X = param_old.abs() ** (2 - self.p_norm) - update_term = X / (X + self.p_norm * group['lr'] * lambda_p_group) + update_term = X / (X + self.p_norm * group["lr"] * lambda_p_group) # pWD step param.data.mul_(update_term) return loss diff --git a/src/pquant/core/torch/pruning_methods/metric_functions.py b/src/pquant/core/torch/pruning_methods/metric_functions.py index e005c5c..054f338 100644 --- a/src/pquant/core/torch/pruning_methods/metric_functions.py +++ b/src/pquant/core/torch/pruning_methods/metric_functions.py @@ -4,17 +4,17 @@ class UnstructuredSparsityMetric: """L0-L1 based metric — torch port of the keras version.""" - def __init__(self, l0_mode='coarse', scale_mode="mean", epsilon=1e-3, target_sparsity=0.8, alpha=100.0): - assert l0_mode in ['coarse', 'smooth'], "Mode must be 'coarse' or 'smooth'" - assert scale_mode in ['sum', 'mean'], "Scale mode must be 'sum' or 'mean'" + def __init__(self, l0_mode="coarse", scale_mode="mean", epsilon=1e-3, target_sparsity=0.8, alpha=100.0): + assert l0_mode in ["coarse", "smooth"], "Mode must be 'coarse' or 'smooth'" + assert scale_mode in ["sum", "mean"], "Scale mode must be 'sum' or 'mean'" assert 0 <= target_sparsity <= 1, "target_sparsity must be between 0 and 1" self.l0_mode = l0_mode self.scale_mode = scale_mode self.target_sparsity = float(target_sparsity) self.epsilon = float(epsilon) self.alpha = float(alpha) - self.l0_fn = self._coarse_l0 if l0_mode == 'coarse' else self._smooth_l0 - self._scaling = self._mean_scaling if scale_mode == 'mean' else self._sum_scaling + self.l0_fn = self._coarse_l0 if l0_mode == "coarse" else self._smooth_l0 + self._scaling = self._mean_scaling if scale_mode == "mean" else self._sum_scaling def _sum_scaling(self, fn_value, num): return fn_value diff --git a/src/pquant/data_models/fitcompress_model.py b/src/pquant/data_models/fitcompress_model.py index 8ea61d6..f0b0124 100644 --- a/src/pquant/data_models/fitcompress_model.py +++ b/src/pquant/data_models/fitcompress_model.py @@ -1,5 +1,3 @@ -from typing import List - from pydantic import BaseModel, Field @@ -12,7 +10,7 @@ class PruningSchedule(BaseModel): class BaseFitCompressModel(BaseModel): enable_fitcompress: bool = Field(default=False) optimize_quantization: bool = Field(default=True) - quantization_schedule: List[float] = Field(default_factory=lambda: [7.0, 4.0, 3.0, 2.0]) + quantization_schedule: list[float] = Field(default_factory=lambda: [7.0, 4.0, 3.0, 2.0]) pruning_schedule: PruningSchedule = Field(default_factory=PruningSchedule) compression_goal: float = Field(default=0.10) optimize_pruning: bool = Field(default=False) diff --git a/src/pquant/data_models/hyperparameter_optimization_model.py b/src/pquant/data_models/hyperparameter_optimization_model.py index 1c53ca7..258ecd9 100644 --- a/src/pquant/data_models/hyperparameter_optimization_model.py +++ b/src/pquant/data_models/hyperparameter_optimization_model.py @@ -1,16 +1,16 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Any from pydantic import BaseModel, Field class HyperparameterSearch(BaseModel): - numerical: Dict[str, List[Union[int, float]]] = Field(default_factory=dict) - categorical: Optional[Dict[str, List[str]]] = Field(default_factory=dict) + numerical: dict[str, list[int | float]] = Field(default_factory=dict) + categorical: dict[str, list[str]] | None = Field(default_factory=dict) class Sampler(BaseModel): type: str = Field(default="TPESampler") - params: Dict[str, Any] = Field(default_factory=dict) + params: dict[str, Any] = Field(default_factory=dict) class BaseHyperparameterOptimizationModel(BaseModel): diff --git a/src/pquant/data_models/pruning_model.py b/src/pquant/data_models/pruning_model.py index 21e6607..11b797a 100644 --- a/src/pquant/data_models/pruning_model.py +++ b/src/pquant/data_models/pruning_model.py @@ -1,11 +1,11 @@ from enum import Enum -from typing import List, Literal, Optional +from typing import Literal from pydantic import BaseModel, Field class BasePruningModel(BaseModel): - disable_pruning_for_layers: List[str] = Field(default_factory=list) + disable_pruning_for_layers: list[str] = Field(default_factory=list) enable_pruning: bool = Field(default=True) threshold_decay: float = Field(default=0.0) @@ -39,8 +39,8 @@ class PDPPruningModel(BasePruningModel): class WandaPruningModel(BasePruningModel): pruning_method: Literal["wanda"] = "wanda" - M: Optional[int] = (Field(default=None),) - N: Optional[int] = (Field(default=None),) + M: int | None = (Field(default=None),) + N: int | None = (Field(default=None),) sparsity: float = Field(default=0.9) t_delta: int = Field(default=100) t_start_collecting_batch: int = Field(default=100) diff --git a/src/pquant/data_models/training_model.py b/src/pquant/data_models/training_model.py index 1619b59..abd70a9 100644 --- a/src/pquant/data_models/training_model.py +++ b/src/pquant/data_models/training_model.py @@ -2,7 +2,7 @@ class BaseTrainingModel(BaseModel): - model_config = ConfigDict(extra='allow') + model_config = ConfigDict(extra="allow") epochs: int = Field(default=200) fine_tuning_epochs: int = Field(default=0) pretraining_epochs: int = Field(default=50) diff --git a/tests/conftest.py b/tests/conftest.py index 8e377b0..c26d7ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ def set_image_data_format(): keras.backend.set_image_data_format(os.environ.get("DATA_FORMAT", "channels_last")) -@pytest.fixture(scope='session', autouse=True, params=[42]) +@pytest.fixture(scope="session", autouse=True, params=[42]) def set_random_seed(request): """Set random seeds for reproducibility""" @@ -20,36 +20,36 @@ def set_random_seed(request): random.seed(seed) backend = keras.backend.backend() match backend: - case 'tensorflow': + case "tensorflow": import tensorflow as tf tf.random.set_seed(seed) - case 'torch': + case "torch": import torch torch.manual_seed(seed) case _: - raise ValueError(f'Unknown backend: {backend}') + raise ValueError(f"Unknown backend: {backend}") -@pytest.fixture(scope='session', autouse=True) +@pytest.fixture(scope="session", autouse=True) def configure_backend(): backend = keras.backend.backend() match backend: - case 'tensorflow': + case "tensorflow": import tensorflow as tf # Use full float32 matmul precision to match numpy/onnxruntime. Without this, TF uses # TF32 on Ampere+ GPUs and the ~1e-3 relative error exceeds tight test tolerances # (e.g. the keras→ONNX parity tests). Mirrors the torch 'highest' setting below. tf.config.experimental.enable_tensor_float_32_execution(False) - case 'torch': + case "torch": import torch - torch.set_float32_matmul_precision('highest') + torch.set_float32_matmul_precision("highest") device = "cuda" if torch.cuda.is_available() else "cpu" torch.set_default_device(device) torch.set_default_dtype(torch.float32) case _: - raise ValueError(f'Unknown backend: {backend}') + raise ValueError(f"Unknown backend: {backend}") diff --git a/tests/test_hgq_keras.py b/tests/test_hgq_keras.py index a4dcf51..780891e 100644 --- a/tests/test_hgq_keras.py +++ b/tests/test_hgq_keras.py @@ -15,9 +15,6 @@ import numpy as np import pytest from keras import ops - -from pquant import pdp_config -from pquant.core.keras.quantizer import Quantizer from pquant.layers import ( PQAvgPool1d, PQBatchNormalization, @@ -27,6 +24,9 @@ PQMultiheadAttention, ) +from pquant import pdp_config +from pquant.core.keras.quantizer import Quantizer + BATCH_SIZE = 4 IN_FEATURES = 16 OUT_FEATURES = 32 diff --git a/tests/test_hgq_torch.py b/tests/test_hgq_torch.py index c039ced..456d6aa 100644 --- a/tests/test_hgq_torch.py +++ b/tests/test_hgq_torch.py @@ -89,7 +89,7 @@ def test_forward_matches_keras(overflow, round_mode, is_data): assert out_torch.shape == out_keras.shape, f"shape mismatch: {out_torch.shape} vs {out_keras.shape}" assert torch.allclose(out_torch, out_keras, rtol=RTOL, atol=ATOL), ( - f"[{overflow}/{round_mode}/is_data={is_data}] " f"max diff = {(out_torch - out_keras).abs().max().item():.6g}" + f"[{overflow}/{round_mode}/is_data={is_data}] max diff = {(out_torch - out_keras).abs().max().item():.6g}" ) @@ -103,7 +103,7 @@ def test_forward_matches_keras_training_sat(): out_keras = _as_torch(keras_q(x, training=True)).detach() assert torch.allclose(out_torch, out_keras, rtol=RTOL, atol=ATOL), ( - f"training forward diverged, max diff = " f"{(out_torch - out_keras).abs().max().item():.6g}" + f"training forward diverged, max diff = {(out_torch - out_keras).abs().max().item():.6g}" ) @@ -139,7 +139,7 @@ def test_backward_f_gradient_sat(): assert grad_f_torch.shape == grad_f_keras.shape, f"grad f shape mismatch: {grad_f_torch.shape} vs {grad_f_keras.shape}" # Gradient direction/magnitude should match up to STE discretisation noise. assert torch.allclose(grad_f_torch, grad_f_keras, rtol=1e-3, atol=1e-5), ( - f"grad f mismatch, max diff = " f"{(grad_f_torch - grad_f_keras).abs().max().item():.6g}" + f"grad f mismatch, max diff = {(grad_f_torch - grad_f_keras).abs().max().item():.6g}" ) @@ -154,7 +154,7 @@ def test_backward_input_gradient_ste(): assert x.grad is not None assert torch.allclose(x.grad, torch.ones_like(x), atol=1e-5), ( - f"STE grad should be ~1 inside sat range, got max deviation " f"{(x.grad - 1).abs().max().item():.6g}" + f"STE grad should be ~1 inside sat range, got max deviation {(x.grad - 1).abs().max().item():.6g}" ) diff --git a/tests/test_keras_alkaid_conversion.py b/tests/test_keras_alkaid_conversion.py index 04a866b..8285169 100644 --- a/tests/test_keras_alkaid_conversion.py +++ b/tests/test_keras_alkaid_conversion.py @@ -6,11 +6,7 @@ from alkaid.codegen import RTLModel # noqa: E402 from alkaid.converter import trace_model from alkaid.trace import trace # noqa: E402 - -from pquant import pdp_config -from pquant._alkaid_plugin import _alkaid_keras_plugin # noqa: E402 from pquant.activations import PQActivation -from pquant.core.keras.quantizer import Quantizer from pquant.layers import ( PQAvgPool1d, PQAvgPool2d, @@ -25,6 +21,10 @@ apply_final_compression, ) +from pquant import pdp_config +from pquant._alkaid_plugin import _alkaid_keras_plugin # noqa: E402 +from pquant.core.keras.quantizer import Quantizer + _alkaid_keras_plugin.register() IN_FEATURES = 3 diff --git a/tests/test_keras_compression_layers.py b/tests/test_keras_compression_layers.py index 1d3be67..4f321b4 100644 --- a/tests/test_keras_compression_layers.py +++ b/tests/test_keras_compression_layers.py @@ -15,18 +15,7 @@ ReLU, SeparableConv2D, ) - -from pquant import ( - ap_config, - autosparse_config, - cs_config, - dst_config, - mdmm_config, - pdp_config, - wanda_config, -) from pquant.activations import PQActivation -from pquant.core.hyperparameter_optimization import PQConfig from pquant.layers import ( PQAvgPool1d, PQAvgPool2d, @@ -44,6 +33,17 @@ pre_finetune_functions, ) +from pquant import ( + ap_config, + autosparse_config, + cs_config, + dst_config, + mdmm_config, + pdp_config, + wanda_config, +) +from pquant.core.hyperparameter_optimization import PQConfig + BATCH_SIZE = 4 OUT_FEATURES = 32 IN_FEATURES = 16 @@ -1469,13 +1469,13 @@ def test_set_activation_custom_bits_hgq(config_pdp, conv2d_input): assert ops.all(f_input == 7.0) config_pdp.quantization_parameters.layer_specific = { - 'conv2d': { - 'weight': {'integer_bits': 1.0, 'fractional_bits': 3.0}, - 'bias': {'integer_bits': 2.0, 'fractional_bits': 4.0}, + "conv2d": { + "weight": {"integer_bits": 1.0, "fractional_bits": 3.0}, + "bias": {"integer_bits": 2.0, "fractional_bits": 4.0}, }, - 're_lu': {"input": {'integer_bits': 1.0, 'fractional_bits': 3.0}}, - 'average_pooling2d': {"input": {'integer_bits': 1.0, 'fractional_bits': 3.0}}, - 'activation': {"input": {'integer_bits': 0.0, 'fractional_bits': 3.0}}, + "re_lu": {"input": {"integer_bits": 1.0, "fractional_bits": 3.0}}, + "average_pooling2d": {"input": {"integer_bits": 1.0, "fractional_bits": 3.0}}, + "activation": {"input": {"integer_bits": 0.0, "fractional_bits": 3.0}}, } keras.backend.clear_session() inputs = keras.Input(shape=conv2d_input.shape[1:]) @@ -1536,13 +1536,13 @@ def test_set_activation_custom_bits_quantizer(config_pdp, conv2d_input): assert m.f_input == 7.0 config_pdp.quantization_parameters.layer_specific = { - 'conv2d': { - 'weight': {'integer_bits': 1.0, 'fractional_bits': 3.0}, - 'bias': {'integer_bits': 2.0, 'fractional_bits': 4.0}, + "conv2d": { + "weight": {"integer_bits": 1.0, "fractional_bits": 3.0}, + "bias": {"integer_bits": 2.0, "fractional_bits": 4.0}, }, - 're_lu': {"input": {'integer_bits': 1.0, 'fractional_bits': 3.0}}, - 'average_pooling2d': {"input": {'integer_bits': 1.0, 'fractional_bits': 3.0}}, - 'activation': {"input": {'integer_bits': 0.0, 'fractional_bits': 3.0}}, + "re_lu": {"input": {"integer_bits": 1.0, "fractional_bits": 3.0}}, + "average_pooling2d": {"input": {"integer_bits": 1.0, "fractional_bits": 3.0}}, + "activation": {"input": {"integer_bits": 0.0, "fractional_bits": 3.0}}, } keras.backend.clear_session() inputs = keras.Input(shape=conv2d_input.shape[1:]) @@ -1710,7 +1710,6 @@ def test_avg_pool1d(config_pdp, conv1d_input): class DummyLayer(keras.layers.Layer): - def __init__(self, *args, **kwargs): super().__init__() self.built = True @@ -1735,7 +1734,7 @@ def extra_repr(self): def test_avgpool_quant_called(config_pdp, conv1d_input): config_pdp.quantization_parameters.enable_quantization = True - with patch('pquant.layers.Quantizer', DummyLayer): + with patch("pquant.layers.Quantizer", DummyLayer): layer = PQAvgPool1d(config_pdp, KERNEL_SIZE, quantize_input=True) layer(conv1d_input) assert layer.input_quantizer.layer_called == 1 @@ -1757,7 +1756,7 @@ def test_avgpool_quant_called(config_pdp, conv1d_input): def test_batchnorm_quant_called(config_pdp, conv2d_input): config_pdp.quantization_parameters.enable_quantization = True axis = -1 if keras.backend.image_data_format() == "channels_last" else 1 - with patch('pquant.layers.Quantizer', DummyLayer): + with patch("pquant.layers.Quantizer", DummyLayer): layer = PQBatchNormalization(config_pdp, axis=axis, quantize_input=True) layer(conv2d_input) assert layer.input_quantizer.layer_called == 1 @@ -1781,7 +1780,7 @@ def test_batchnorm_quant_called(config_pdp, conv2d_input): def test_pqconv2d_quant_called(config_pdp, conv2d_input): config_pdp.quantization_parameters.enable_quantization = True - with patch('pquant.layers.Quantizer', DummyLayer): + with patch("pquant.layers.Quantizer", DummyLayer): layer = PQConv2d(config_pdp, OUT_FEATURES, KERNEL_SIZE, quantize_input=True, use_bias=True) layer.post_pre_train_function() layer(conv2d_input) @@ -1812,7 +1811,7 @@ def test_pqconv2d_quant_called(config_pdp, conv2d_input): def test_pqdepthwiseconv2d_quant_called(config_pdp, conv2d_input): config_pdp.quantization_parameters.enable_quantization = True - with patch('pquant.layers.Quantizer', DummyLayer): + with patch("pquant.layers.Quantizer", DummyLayer): layer = PQDepthwiseConv2d(config_pdp, KERNEL_SIZE, quantize_input=True, use_bias=True) layer.post_pre_train_function() layer(conv2d_input) @@ -1842,7 +1841,7 @@ def test_pqdepthwiseconv2d_quant_called(config_pdp, conv2d_input): def test_pqconv1d_quant_called(config_pdp, conv1d_input): config_pdp.quantization_parameters.enable_quantization = True - with patch('pquant.layers.Quantizer', DummyLayer): + with patch("pquant.layers.Quantizer", DummyLayer): layer = PQConv1d(config_pdp, OUT_FEATURES, KERNEL_SIZE, quantize_input=True, use_bias=True) layer.post_pre_train_function() layer(conv1d_input) @@ -1872,7 +1871,7 @@ def test_pqconv1d_quant_called(config_pdp, conv1d_input): def test_dense_quant_called(config_pdp, dense_input): config_pdp.quantization_parameters.enable_quantization = True - with patch('pquant.layers.Quantizer', DummyLayer): + with patch("pquant.layers.Quantizer", DummyLayer): layer = PQDense(config_pdp, OUT_FEATURES, quantize_input=True, use_bias=True) layer.post_pre_train_function() layer(dense_input) @@ -1905,7 +1904,7 @@ def test_layer_replacement_quant_called(config_pdp, conv2d_input): config_pdp.quantization_parameters.quantize_input = True config_pdp.quantization_parameters.quantize_output = True config_pdp.quantization_parameters.use_high_granularity_quantization = True - with patch('pquant.layers.Quantizer', DummyLayer): + with patch("pquant.layers.Quantizer", DummyLayer): inp = keras.Input(shape=conv2d_input.shape[1:]) x = Conv2D(OUT_FEATURES, KERNEL_SIZE)(inp) diff --git a/tests/test_torch_alkaid_conversion.py b/tests/test_torch_alkaid_conversion.py index dfeafd1..c9097a2 100644 --- a/tests/test_torch_alkaid_conversion.py +++ b/tests/test_torch_alkaid_conversion.py @@ -6,8 +6,11 @@ import torch.nn as nn from alkaid.codegen import RTLModel # noqa: E402 from alkaid.converter import trace_model -from alkaid.trace import trace # noqa: E402 -from alkaid.trace import FVArray, HWConfig # noqa: E402 +from alkaid.trace import ( # noqa: E402 + FVArray, + HWConfig, + trace, # noqa: E402 +) from pquant import pdp_config from pquant._alkaid_plugin import _alkaid_torch_plugin # noqa: E402 @@ -40,7 +43,6 @@ class TwoBranchNet(nn.Module): - def __init__(self, config): super().__init__() self.conv2d = PQConv2d(config, IN_FEATURES, OUT_FEATURES, KERNEL_SIZE, padding="same") diff --git a/tests/test_torch_checkpoint.py b/tests/test_torch_checkpoint.py index 0c664d2..29624a7 100644 --- a/tests/test_torch_checkpoint.py +++ b/tests/test_torch_checkpoint.py @@ -7,7 +7,6 @@ os.environ["KERAS_BACKEND"] = "torch" -from pquant import dst_config # noqa: E402 from pquant.activations import PQActivation # noqa: E402 from pquant.layers import ( # noqa: E402 PQAvgPool1d, @@ -24,6 +23,8 @@ pre_finetune_functions, ) +from pquant import dst_config # noqa: E402 + BATCH_SIZE = 2 OUT_FEATURES = 8 IN_FEATURES = 4 @@ -32,7 +33,6 @@ class SingleLayerModel(nn.Module): - def __init__(self, layer, is_mha=False): super().__init__() self.layer = layer diff --git a/tests/test_torch_compression_layers.py b/tests/test_torch_compression_layers.py index e39ed66..778ef73 100644 --- a/tests/test_torch_compression_layers.py +++ b/tests/test_torch_compression_layers.py @@ -18,10 +18,7 @@ import keras # noqa: E402 from keras import ops # noqa: E402 - -from pquant import post_training_prune # noqa: E402 from pquant.activations import PQActivation # noqa: E402 -from pquant.core.hyperparameter_optimization import PQConfig # noqa: E402 from pquant.layers import ( # noqa: E402 PQAvgPool1d, PQAvgPool2d, @@ -38,6 +35,9 @@ pre_finetune_functions, ) +from pquant import post_training_prune # noqa: E402 +from pquant.core.hyperparameter_optimization import PQConfig # noqa: E402 + BATCH_SIZE = 4 OUT_FEATURES = 32 IN_FEATURES = 16 @@ -668,13 +668,13 @@ def test_set_activation_custom_bits_hgq(config_pdp, conv2d_input): assert torch.all(m.input_quantizer.quantizer.f == 7.0) config_pdp.quantization_parameters.layer_specific = { - 'submodule': { - 'weight': {'integer_bits': 1, 'fractional_bits': 3}, - 'bias': {'integer_bits': 2, 'fractional_bits': 4}, + "submodule": { + "weight": {"integer_bits": 1, "fractional_bits": 3}, + "bias": {"integer_bits": 2, "fractional_bits": 4}, }, - 'submodule2': {"input": {'integer_bits': 1, 'fractional_bits': 3}}, - 'activation': {"input": {'integer_bits': 1, 'fractional_bits': 4}}, - 'activation2': {"input": {'integer_bits': 0, 'fractional_bits': 3}}, + "submodule2": {"input": {"integer_bits": 1, "fractional_bits": 3}}, + "activation": {"input": {"integer_bits": 1, "fractional_bits": 4}}, + "activation2": {"input": {"integer_bits": 0, "fractional_bits": 3}}, } model = TestModel2(layer, layer2, "relu", "tanh") @@ -748,13 +748,13 @@ def test_set_activation_custom_bits_quantizer(config_pdp, conv2d_input): assert m.f_input == 8.0 config_pdp.quantization_parameters.layer_specific = { - 'submodule': { - 'weight': {'integer_bits': 1.0, 'fractional_bits': 3.0}, - 'bias': {'integer_bits': 1.0, 'fractional_bits': 3.0}, + "submodule": { + "weight": {"integer_bits": 1.0, "fractional_bits": 3.0}, + "bias": {"integer_bits": 1.0, "fractional_bits": 3.0}, }, - 'submodule2': {"input": {'integer_bits': 1.0, 'fractional_bits': 3.0}}, - 'activation': {"input": {'integer_bits': 0.0, 'fractional_bits': 4.0}}, - 'activation2': {"input": {'integer_bits': 0.0, 'fractional_bits': 3.0}}, + "submodule2": {"input": {"integer_bits": 1.0, "fractional_bits": 3.0}}, + "activation": {"input": {"integer_bits": 0.0, "fractional_bits": 4.0}}, + "activation2": {"input": {"integer_bits": 0.0, "fractional_bits": 3.0}}, } model = TestModel2(layer, layer2, "relu", "tanh") @@ -1175,7 +1175,6 @@ def test_batchnorm2d_direct_hgq(config_pdp, conv2d_input): class DummyLayer(nn.Module): - def __init__(self, is_pretraining=False): super().__init__() self.built = True @@ -1745,7 +1744,6 @@ def dummy_hgq_loss(): class ModelWithAllLayers(nn.Module): - def __init__(self, use_bias=True): super().__init__() self.conv = Conv2d(IN_FEATURES, OUT_FEATURES, KERNEL_SIZE, bias=use_bias) diff --git a/tests/test_torch_missing_quantizer_tracing.py b/tests/test_torch_missing_quantizer_tracing.py index 60edf4f..635a9fb 100644 --- a/tests/test_torch_missing_quantizer_tracing.py +++ b/tests/test_torch_missing_quantizer_tracing.py @@ -7,10 +7,11 @@ os.environ["KERAS_BACKEND"] = "torch" from pquant.activations import PQActivation # noqa: E402 +from pquant.layers import PQDense # noqa: E402 + from pquant.core.hyperparameter_optimization import PQConfig # noqa: E402 from pquant.core.torch.quantizer import Quantizer # noqa: E402 from pquant.core.torch.tracing import check_quantization # noqa: E402 -from pquant.layers import PQDense # noqa: E402 BATCH_SIZE = 4 OUT_FEATURES = 32 diff --git a/tests/test_torch_onnx_converter.py b/tests/test_torch_onnx_converter.py index 2064d30..c3570bd 100644 --- a/tests/test_torch_onnx_converter.py +++ b/tests/test_torch_onnx_converter.py @@ -17,12 +17,6 @@ os.environ["KERAS_BACKEND"] = "torch" -import pquant # noqa: E402 -from pquant.core.torch.convert_to_onnx import ( # noqa: E402 - convert_to_onnx, - convert_to_onnx_fx, - export_qdq_layernorm, -) from pquant.layers import ( # noqa: E402 PQAvgPool1d, PQAvgPool2d, @@ -34,6 +28,13 @@ PQMultiheadAttention, ) +import pquant # noqa: E402 +from pquant.core.torch.convert_to_onnx import ( # noqa: E402 + convert_to_onnx, + convert_to_onnx_fx, + export_qdq_layernorm, +) + ort = pytest.importorskip("onnxruntime", reason="onnxruntime not installed") ATOL = 1e-4 # float32 Gemm/Conv can differ by ~1 ULP; keep some slack diff --git a/tests/test_torch_pruning_layers.py b/tests/test_torch_pruning_layers.py index c7e7b59..88a8eb3 100644 --- a/tests/test_torch_pruning_layers.py +++ b/tests/test_torch_pruning_layers.py @@ -249,7 +249,7 @@ def test_pdp_matches_keras(layer_type, shape, structured): t_mask_np = to_numpy(t_layer.mask) actual_sparsity = float((t_mask_np < 0.5).sum()) / t_mask_np.size assert actual_sparsity == pytest.approx(target_sparsity, abs=1e-6), ( - f"PDP {layer_type} (structured={structured}) mask sparsity " f"{actual_sparsity} != target {target_sparsity}" + f"PDP {layer_type} (structured={structured}) mask sparsity {actual_sparsity} != target {target_sparsity}" ) @@ -430,9 +430,9 @@ def test_wanda_matches_keras(layer_type, shape, N, M): target_sparsity = (N / M) if (N is not None and M is not None) else cfg["pruning_parameters"]["sparsity"] mask_np = to_numpy(t_layer.mask) pruned_fraction = float((mask_np == 0).sum()) / mask_np.size - assert pruned_fraction == pytest.approx( - target_sparsity - ), f"Wanda {layer_type} (N={N}, M={M}) pruned fraction {pruned_fraction} != target {target_sparsity}" + assert pruned_fraction == pytest.approx(target_sparsity), ( + f"Wanda {layer_type} (N={N}, M={M}) pruned fraction {pruned_fraction} != target {target_sparsity}" + ) k_weight = keras_tensor(w_np) t_weight = torch_tensor(w_np, requires_grad=True) From 1ac8ad8112746b337ba123809c1e52a409367c3d Mon Sep 17 00:00:00 2001 From: Roope Niemi Date: Thu, 30 Jul 2026 16:53:29 +0200 Subject: [PATCH 20/22] add git blame ignore revs --- .git-blame-ignore-revs | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000..69686f2 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,5 @@ +# Commits to ignore in git blame (formatting-only changes). +# GitHub picks this file up automatically; locally, enable it with: +# git config blame.ignoreRevsFile .git-blame-ignore-revs + +eb65ffb809108b784e2051767f0c0f3f9e672fb3 From 4394f48c3b484abedb2f71831255d11ad6757da1 Mon Sep 17 00:00:00 2001 From: Roope Niemi Date: Thu, 30 Jul 2026 16:27:01 +0200 Subject: [PATCH 21/22] loss fix for cs and dst --- src/pquant/core/keras/pruning_methods/cs.py | 5 ++++- src/pquant/core/keras/pruning_methods/dst.py | 6 +++--- src/pquant/core/torch/pruning_methods/cs.py | 2 ++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/pquant/core/keras/pruning_methods/cs.py b/src/pquant/core/keras/pruning_methods/cs.py index 97e64ff..eddf561 100644 --- a/src/pquant/core/keras/pruning_methods/cs.py +++ b/src/pquant/core/keras/pruning_methods/cs.py @@ -45,6 +45,7 @@ def call(self, weight): use_current_mask = ops.logical_or(self.is_pretraining, self.is_finetuning) updated_mask = ops.where(use_current_mask, stored_mask, new_mask) self.mask.assign(updated_mask) + self.add_loss(self.calculate_additional_loss()) return updated_mask * weight def pre_finetune_function(self): @@ -82,9 +83,11 @@ def post_round_function(self): self.beta.assign(1.0) def calculate_additional_loss(self): - return ops.convert_to_tensor( + penalty = ops.convert_to_tensor( self.config.pruning_parameters.threshold_decay * ops.norm(ops.ravel(self.get_mask()), ord=1) ) + inactive = ops.logical_or(self.is_pretraining, self.is_finetuning) + return ops.where(inactive, ops.zeros_like(penalty), penalty) def get_layer_sparsity(self, weight): return ops.sum(self.get_hard_mask()) / ops.size(weight) diff --git a/src/pquant/core/keras/pruning_methods/dst.py b/src/pquant/core/keras/pruning_methods/dst.py index e2d5ca9..c040db9 100644 --- a/src/pquant/core/keras/pruning_methods/dst.py +++ b/src/pquant/core/keras/pruning_methods/dst.py @@ -108,9 +108,9 @@ def get_layer_sparsity(self, weight): return ops.sum(self.get_mask(weight)) / ops.size(weight) def calculate_additional_loss(self): - if self._is_pretraining or self._is_finetuning: - return ops.cast(0.0, self.threshold.dtype) - return self.config.pruning_parameters.alpha * ops.sum(ops.exp(-self.threshold)) + penalty = self.config.pruning_parameters.alpha * ops.sum(ops.exp(-self.threshold)) + inactive = ops.logical_or(self.is_pretraining, self.is_finetuning) + return ops.where(inactive, ops.zeros_like(penalty), penalty) def pre_finetune_function(self): self._is_finetuning = True diff --git a/src/pquant/core/torch/pruning_methods/cs.py b/src/pquant/core/torch/pruning_methods/cs.py index 3881ebc..15f9f53 100644 --- a/src/pquant/core/torch/pruning_methods/cs.py +++ b/src/pquant/core/torch/pruning_methods/cs.py @@ -73,6 +73,8 @@ def post_round_function(self): self.beta.fill_(1.0) def calculate_additional_loss(self): + if self._is_pretraining or self._is_finetuning: + return torch.zeros((), dtype=self.s.dtype, device=self.s.device) return self.config.pruning_parameters.threshold_decay * torch.norm(self.get_mask().reshape(-1), p=1) def get_layer_sparsity(self, weight): From 111529ac5ac2beb54df85bca628d85295c818090 Mon Sep 17 00:00:00 2001 From: Roope Niemi Date: Fri, 31 Jul 2026 15:14:35 +0200 Subject: [PATCH 22/22] fix keras mha sep-conv bug, fix torch hgq hls4ml conversion bug --- src/pquant/core/keras/activations.py | 15 +++++-- src/pquant/core/keras/layers.py | 54 ++++++++++++++++++++++++-- src/pquant/core/torch/quantizer.py | 24 +++++++++--- tests/test_keras_compression_layers.py | 9 ++++- 4 files changed, 88 insertions(+), 14 deletions(-) diff --git a/src/pquant/core/keras/activations.py b/src/pquant/core/keras/activations.py index feb307d..fccbe8b 100644 --- a/src/pquant/core/keras/activations.py +++ b/src/pquant/core/keras/activations.py @@ -340,9 +340,18 @@ def _data_quantizer(k, i, f): self.input_quantizer = _data_quantizer(self.k_input, self.i_input, self.f_input) self.output_quantizer = _data_quantizer(self.k_output, self.i_output, self.f_output) - if self.use_hgq: - self.input_quantizer.build(input_shape) - self.output_quantizer.build(input_shape) + accum_shape = tuple(1 if i in self.axes else s for i, s in enumerate(input_shape)) + self.input_quantizer.build(input_shape) + self.output_quantizer.build(input_shape) + if not self.exp_table.built: + self.exp_table.build(input_shape) + if not self.inv_table.built: + self.inv_table.build(accum_shape) + for table, shape in ((self.exp_table, input_shape), (self.inv_table, accum_shape)): + if table.quantize_input and not table.input_quantizer.built: + table.input_quantizer.build(shape) + if table.quantize_output and not table.output_quantizer.built: + table.output_quantizer.build(shape) super().build(input_shape) def get_input_quantization_bits(self): diff --git a/src/pquant/core/keras/layers.py b/src/pquant/core/keras/layers.py index c607c79..e602cb5 100644 --- a/src/pquant/core/keras/layers.py +++ b/src/pquant/core/keras/layers.py @@ -846,6 +846,16 @@ def _rewind_weights(self): self.depthwise_conv._rewind_weights() self.pointwise_conv._rewind_weights() + def build(self, input_shape): + self.depthwise_conv.build(input_shape) + intermediate_shape = self.depthwise_conv.compute_output_shape(input_shape) + self.pointwise_conv.build(intermediate_shape) + super().build(input_shape) + + def compute_output_shape(self, input_shape): + intermediate_shape = self.depthwise_conv.compute_output_shape(input_shape) + return self.pointwise_conv.compute_output_shape(intermediate_shape) + def call(self, x, training=None): x = self.depthwise_conv(x, training=training) x = self.pointwise_conv(x, training=training) @@ -1811,6 +1821,42 @@ def hgq_loss(self): return ops.convert_to_tensor(0.0) return ops.convert_to_tensor(self.hgq_beta * self.attention_ebops() + self.softmax.hgq_loss()) + def build(self, input_shape): + if isinstance(input_shape, (list, tuple)) and isinstance(input_shape[0], (list, tuple)): + if len(input_shape) == 3: + query_shape, key_shape, value_shape = input_shape + elif len(input_shape) == 2: + query_shape, key_shape = input_shape + value_shape = key_shape + else: + query_shape = key_shape = value_shape = input_shape[0] + else: + query_shape = key_shape = value_shape = input_shape + if not self.q_proj.built: + self.q_proj.build(query_shape) + if not self.k_proj.built: + self.k_proj.build(key_shape) + if not self.v_proj.built: + self.v_proj.build(value_shape) + scores_shape = (query_shape[0], self.num_heads, query_shape[1], key_shape[1]) + if not self.softmax.built: + self.softmax.build(scores_shape) + if not self.out_proj.built: + self.out_proj.build((query_shape[0], query_shape[1], self.embed_dim)) + super().build(input_shape) + + def compute_output_spec(self, inputs, training=None, key_padding_mask=None, attn_mask=None, need_weights=True): + if isinstance(inputs, (list, tuple)): + query = inputs[0] + key = inputs[1] if len(inputs) > 1 else inputs[0] + else: + query = key = inputs + out = keras.KerasTensor((query.shape[0], query.shape[1], self.embed_dim), dtype=self.compute_dtype) + if need_weights: + attn = keras.KerasTensor((query.shape[0], query.shape[1], key.shape[1]), dtype=self.compute_dtype) + return out, attn + return out, None + def call( self, inputs, @@ -2222,10 +2268,12 @@ def add_compression_layers(model, config, input_shape=None): new_layer.pointwise_conv.set_enable_pruning(enable_pruning_pointwise) _build_pruning_layer_from_kernel(new_layer.depthwise_conv, layer.depthwise_kernel) _build_pruning_layer_from_kernel(new_layer.pointwise_conv, layer.pointwise_kernel) - new_layer.depthwise_conv.build(x.shape) - y = new_layer.depthwise_conv(x).shape - new_layer.pointwise_conv.build(y) + new_layer.build(x.shape) x = new_layer(x) + new_layer.depthwise_conv._kernel.assign(layer.depthwise_kernel) + new_layer.pointwise_conv._kernel.assign(layer.pointwise_kernel) + if layer.use_bias: + new_layer.pointwise_conv._bias.assign(layer.bias) act = _check_activation(layer, config) elif isinstance(layer, Conv1D): new_layer = PQConv1d( diff --git a/src/pquant/core/torch/quantizer.py b/src/pquant/core/torch/quantizer.py index b664547..51f7e38 100644 --- a/src/pquant/core/torch/quantizer.py +++ b/src/pquant/core/torch/quantizer.py @@ -30,12 +30,13 @@ def __init__( self.is_data = is_data self.dynamic_data = dynamic_data self.granularity = QuantizationGranularity(granularity).value - if not self.use_hgq: - param_shape = () if is_data else self.compute_weight_param_shape(shape) - self.k = torch.nn.Parameter(torch.full(param_shape, float(k)), requires_grad=False) - self.i = torch.nn.Parameter(torch.full(param_shape, float(i)), requires_grad=False) - self.f = torch.nn.Parameter(torch.full(param_shape, float(f)), requires_grad=False) - self.b = torch.nn.Parameter(torch.full(param_shape, float(i + k + f)), requires_grad=False) + + # Params even when using HGQ, as they are used during hls4ml conversion + param_shape = () if (is_data or self.use_hgq) else self.compute_weight_param_shape(shape) + self.k = torch.nn.Parameter(torch.full(param_shape, float(k)), requires_grad=False) + self.i = torch.nn.Parameter(torch.full(param_shape, float(i)), requires_grad=False) + self.f = torch.nn.Parameter(torch.full(param_shape, float(f)), requires_grad=False) + self.b = torch.nn.Parameter(torch.full(param_shape, float(i + k + f)), requires_grad=False) self.quantizer = create_quantizer( k, i, @@ -64,6 +65,16 @@ def get_total_bits(self, shape): b = self.i + self.f + self.k return torch.ones(shape).to(b.device) * b + def _sync_hgq_mirror_bits(self): + if not self.quantizer.built: + return + with torch.no_grad(): + k, i, f = self.quantizer.k.detach(), self.quantizer.i.detach(), self.quantizer.f.detach() + self.k.data = k.clone() + self.i.data = i.clone() + self.f.data = f.clone() + self.b.data = k + i + f + def set_quantization_bits(self, i, f): if self.use_hgq: self.quantizer.set_bits(i, f) @@ -146,6 +157,7 @@ def apply_final_compression(self): self.quantizer._f.data.clamp_(self.quantizer.f_min, self.quantizer.f_max) if self.quantizer.overflow_mode != "WRAP": self.quantizer._i.data.clamp_(self.quantizer.i_min, self.quantizer.i_max) + self._sync_hgq_mirror_bits() self.final_compression_done.fill_(True) return _, i, f = self.get_quantization_bits() diff --git a/tests/test_keras_compression_layers.py b/tests/test_keras_compression_layers.py index 4f321b4..6df0073 100644 --- a/tests/test_keras_compression_layers.py +++ b/tests/test_keras_compression_layers.py @@ -302,8 +302,7 @@ def test_separable_conv2d_call(config_pdp, conv2d_input): layer_to_replace.pointwise_constraint, layer_to_replace.bias_constraint, ) - layer.depthwise_conv.build(conv2d_input.shape) - layer.pointwise_conv.build(conv2d_input.shape) + layer.build(conv2d_input.shape) layer.depthwise_conv._kernel.assign(layer_to_replace.depthwise_kernel) layer.pointwise_conv._kernel.assign(layer_to_replace.pointwise_kernel) @@ -317,9 +316,15 @@ def test_separable_conv2d_add_remove_layers(config_pdp, conv2d_input): inputs = keras.Input(shape=conv2d_input.shape[1:]) out = SeparableConv2D(OUT_FEATURES, KERNEL_SIZE, use_bias=False, padding="same")(inputs) model = keras.Model(inputs=inputs, outputs=out, name="test_conv2d") + depthwise_kernel = ops.copy(model.layers[1].depthwise_kernel) + pointwise_kernel = ops.copy(model.layers[1].pointwise_kernel) model = add_compression_layers(model, config_pdp, conv2d_input.shape) model(conv2d_input) + # The original layer's weights must survive the replacement. + assert ops.all(ops.equal(model.layers[1].depthwise_conv._kernel, depthwise_kernel)) + assert ops.all(ops.equal(model.layers[1].pointwise_conv._kernel, pointwise_kernel)) + post_pretrain_functions(model, config_pdp) pre_finetune_functions(model)