From 522b9c9d56bf5313e520c7fdc214327d813b4f9f Mon Sep 17 00:00:00 2001 From: sstruzik Date: Tue, 25 Nov 2025 10:49:53 +0000 Subject: [PATCH 01/10] add pyarrow dtype to oed json --- utils/gen-json.py | 77 ++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/utils/gen-json.py b/utils/gen-json.py index 2e74d17f..67566d4e 100755 --- a/utils/gen-json.py +++ b/utils/gen-json.py @@ -6,36 +6,33 @@ import os import click -pd_converter = { - "0 or 1": "Int8", - "bigint": "Int64", - "binary": "Int8", - "bit": "Int8", - "char": "category", - "date": "category", - "datetime": "category", - "decimal": "float64", - "float": "float64", - "int": "Int64", - "nchar": "category", - "nvarchar": "category", - "real": "float64", - "smalldatetime": "category", - "smallint": "Int32", - "tinyint": "Int32", - "uniqueidentifier": "category", - "varbinary": "bytes", - "varchar": "category", -} - -dtype_to_python = { - "Int8": int, - "Int32": int, - "Int64": int, - "bytes": lambda x: bytes(x, "utf-8"), - "float64": float, - "category": str, -} +# convertion map between OED, numpy_nullable, pyarrow, python +_converter_header = ["OED Data Type", "numpy_nullable", "pyarrow", "python"] +_converter_list = [ + ["0 or 1", "Int8", "int8[pyarrow]", int], + ["bigint", "Int64", "int64[pyarrow]", int], + ["binary", "Int8", "int8[pyarrow]", int], + ["bit", "Int8", "int8[pyarrow]", int], + ["char", "category", "string[pyarrow]", str], + ["date", "category", "string[pyarrow]", str], + ["datetime", "category", "string[pyarrow]", str], + ["decimal", "float64", "float64[pyarrow]", float], + ["float", "float64", "float64[pyarrow]", float], + ["int", "Int64", "int64[pyarrow]", int], + ["nchar", "category", "string[pyarrow]", str], + ["nvarchar", "category", "string[pyarrow]", str], + ["real", "float64", "float64[pyarrow]", float], + ["smalldatetime", "category", "string[pyarrow]", str], + ["smallint", "Int32", "int32[pyarrow]", int], + ["tinyint", "Int32", "int32[pyarrow]", int], + ["uniqueidentifier", "category", "string[pyarrow]", str], + ["varbinary", "bytes", "string[pyarrow]", lambda x: bytes(x, "utf-8")], + ["varchar", "category", "string[pyarrow]", str], +] +converter_df = pd.DataFrame(_converter_list, columns=_converter_header) +pd_converter = converter_df[["OED Data Type", "numpy_nullable"]].set_index("OED Data Type")["numpy_nullable"].to_dict() +pa_converter = converter_df[["OED Data Type", "pyarrow"]].set_index("OED Data Type")["pyarrow"].to_dict() +dtype_to_python = converter_df[["OED Data Type", "python"]].set_index("OED Data Type")["python"].to_dict() # Directory containing the CSV files source_csv_default = pathlib.Path(os.path.dirname(os.path.realpath(__file__))).parent.joinpath('OpenExposureData') @@ -83,7 +80,6 @@ def _read_oed_data(sheet_name, basepath=source_csv_dir): with open(output_path, "w") as fp: json.dump(ods_schema, fp, indent=" ") - print(f"JSON OED specification has been created from dir '{source_csv_dir}'. Output saved to '{output_path}'") @@ -102,20 +98,22 @@ def get_ods_input_fields(ods_fields_df): ods_fields_df = ods_fields_df.assign( pd_dtype=ods_fields_df["Data Type"] .str.split("(", n=1, expand=True)[0] - .map(pd_converter) + .map(pd_converter), + pa_dtype=ods_fields_df["Data Type"] + .str.split("(", n=1, expand=True)[0] + .map(pa_converter) ).rename(columns={"File Name": "File Names"}) ods_fields_df["Case Insensitive Field Name"] = ods_fields_df[ "Input Field Name" ].str.lower() # check that to Data Type is missing from our converter - if ods_fields_df["pd_dtype"].isna().any(): + if ods_fields_df["pd_dtype"].isna().any() or ods_fields_df["pa_dtype"].isna().any(): raise ValueError( - f"missing pd_dtype for:\n" - f"""{ods_fields_df.loc[ods_fields_df['pd_dtype'].isna(), + f"missing dtype for:\n" + f"""{ods_fields_df.loc[(ods_fields_df['pd_dtype'].isna()) | (ods_fields_df["pa_dtype"].isna()), ['File Name', 'Input Field Name', 'Type & Description', 'Data Type']]}""" ) - # split ods_fields per File Name split_df = ods_fields_df["File Names"].str.split(";").apply(pd.Series).stack() split_df = ( @@ -125,7 +123,7 @@ def get_ods_input_fields(ods_fields_df): ods_fields_df["Valid value range"] = ods_fields_df.apply( lambda row: extract_valid_value_range( - row["Valid value range"], dtype_to_python[row["pd_dtype"]] + row["Valid value range"], dtype_to_python[row["Data Type"].split("(", 1)[0]] ), axis=1, ) @@ -271,7 +269,10 @@ def get_cr_field(cr_field_df): cr_field_df = cr_field_df.assign( pd_dtype=cr_field_df["Data Type"] .str.split("(", n=1, expand=True)[0] - .map(pd_converter) + .map(pd_converter), + pa_dtype=cr_field_df["Data Type"] + .str.split("(", n=1, expand=True)[0] + .map(pa_converter) ).rename(columns={"File Name": "File Names"}) # split ods_fields per File Name From a4811465b423f8a9b270e2b2762d61e970179c00 Mon Sep 17 00:00:00 2001 From: sstruzik Date: Wed, 26 Nov 2025 11:08:07 +0000 Subject: [PATCH 02/10] add support backend dtype --- utils/gen-json.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/gen-json.py b/utils/gen-json.py index 67566d4e..4454d474 100755 --- a/utils/gen-json.py +++ b/utils/gen-json.py @@ -60,6 +60,7 @@ def _read_oed_data(sheet_name, basepath=source_csv_dir): ods_schema = {} + ods_schema["backend_dtype"] = ['pd_dtype', 'pa_dtype'] ods_schema["input_fields"] = get_ods_input_fields(_read_oed_data("OEDInputFields")) ods_schema["perils"] = get_ods_perils(_read_oed_data("PerilValues"), _read_oed_data("PerilsCovered")) ods_schema["occupancy"] = get_occupancy(_read_oed_data("OccupancyValues")) From 93fa0b71dbdd67f6c8d512a01f83f0bcd04b4d88 Mon Sep 17 00:00:00 2001 From: sstruzik Date: Thu, 27 Nov 2025 11:02:18 +0000 Subject: [PATCH 03/10] drop 3.8 support --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3274735c..7ae33927 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,7 +97,7 @@ jobs: needs: build_package strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Clone ODS_Tools repo From 797697f99d544a331cfc8b9b271cc6ff87adfe77 Mon Sep 17 00:00:00 2001 From: sstruzik Date: Thu, 27 Nov 2025 11:35:27 +0000 Subject: [PATCH 04/10] set custom ENV_ODS_SCHEMA_PATH --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c188a05..563957c3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,6 +27,7 @@ jobs: SPEC_PATH_IN: './OpenExposureData' JSON_PATH_OUT: './ods_tools/data/OpenExposureData_Spec.json' EXCEL_PATH_OUT: './ods_tools/data/OpenExposureData_Spec.xlsx' + ENV_ODS_SCHEMA_PATH: ${{ env.JSON_PATH_OUT }} steps: - name: Github context From 32a8645b61484deb76cffd309b340c8fc037ecd3 Mon Sep 17 00:00:00 2001 From: sstruzik Date: Thu, 27 Nov 2025 11:56:55 +0000 Subject: [PATCH 05/10] fixup --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 563957c3..0535f40a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: SPEC_PATH_IN: './OpenExposureData' JSON_PATH_OUT: './ods_tools/data/OpenExposureData_Spec.json' EXCEL_PATH_OUT: './ods_tools/data/OpenExposureData_Spec.xlsx' - ENV_ODS_SCHEMA_PATH: ${{ env.JSON_PATH_OUT }} + ENV_ODS_SCHEMA_PATH: './ods_tools/data/OpenExposureData_Spec.json' steps: - name: Github context From d91fd862894242b760db46b0f2e120044bfcbf50 Mon Sep 17 00:00:00 2001 From: sstruzik Date: Thu, 27 Nov 2025 16:20:29 +0000 Subject: [PATCH 06/10] test ci --- .github/workflows/build.yml | 1 - .github/workflows/test.yml | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0535f40a..5c188a05 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,6 @@ jobs: SPEC_PATH_IN: './OpenExposureData' JSON_PATH_OUT: './ods_tools/data/OpenExposureData_Spec.json' EXCEL_PATH_OUT: './ods_tools/data/OpenExposureData_Spec.xlsx' - ENV_ODS_SCHEMA_PATH: './ods_tools/data/OpenExposureData_Spec.json' steps: - name: Github context diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7ae33927..55ee0591 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,3 +125,5 @@ jobs: - name: Run tests run: pytest -v + env: + ENV_ODS_SCHEMA_PATH: ${{ needs.build_spec.outputs.json_filename }} From 9abc4718c38f017a61092619c33030c455ffc513 Mon Sep 17 00:00:00 2001 From: sstruzik Date: Thu, 27 Nov 2025 16:29:54 +0000 Subject: [PATCH 07/10] test ci --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 55ee0591..59184b6a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -126,4 +126,4 @@ jobs: - name: Run tests run: pytest -v env: - ENV_ODS_SCHEMA_PATH: ${{ needs.build_spec.outputs.json_filename }} + ODS_SCHEMA_PATH: ${{ needs.build_spec.outputs.json_filename }} From c5f0b8629a5d42f34b88a21e1ab1abf973c4873a Mon Sep 17 00:00:00 2001 From: Sam Gamble Date: Thu, 27 Nov 2025 16:37:49 +0000 Subject: [PATCH 08/10] retest-ci From 81f1db88cf5a19fdde0cce4c215f2ee4f70f40aa Mon Sep 17 00:00:00 2001 From: sstruzik Date: Thu, 27 Nov 2025 16:40:27 +0000 Subject: [PATCH 09/10] test ci --- .github/workflows/test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 59184b6a..1250e87a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -124,6 +124,9 @@ jobs: run: pip install -r tests/requirements.in - name: Run tests - run: pytest -v env: ODS_SCHEMA_PATH: ${{ needs.build_spec.outputs.json_filename }} + run: | + echo "ODS_SCHEMA_PATH: $ODS_SCHEMA_PATH" + pytest -v + From 255eb0bf38e5844832fbd3686ee9e1c180ebc25c Mon Sep 17 00:00:00 2001 From: sstruzik Date: Thu, 27 Nov 2025 17:07:23 +0000 Subject: [PATCH 10/10] fixup --- .github/workflows/test.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1250e87a..7ae33927 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -124,9 +124,4 @@ jobs: run: pip install -r tests/requirements.in - name: Run tests - env: - ODS_SCHEMA_PATH: ${{ needs.build_spec.outputs.json_filename }} - run: | - echo "ODS_SCHEMA_PATH: $ODS_SCHEMA_PATH" - pytest -v - + run: pytest -v