From 9c9c1f78fc8fe11072c9502b2949c550a7393cfc Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 2 Sep 2026 13:56:25 -0500 Subject: [PATCH 1/7] Cleanup code and fix bugs discovered by additional rules to 'ruff' --- doc/source/conf.py | 2 +- doc/source/toctree_filter.py | 4 ++-- polar2grid/_glue_argparser.py | 10 +++++----- polar2grid/add_coastlines.py | 8 +++----- polar2grid/compare.py | 6 +++--- polar2grid/core/containers.py | 2 +- polar2grid/core/dtype.py | 2 +- polar2grid/core/script_utils.py | 2 +- polar2grid/debug_data.py | 2 +- polar2grid/filters/_utils.py | 2 +- polar2grid/glue.py | 12 ++++++------ polar2grid/grids/config_helper.py | 2 +- polar2grid/grids/manager.py | 2 +- polar2grid/readers/_base.py | 2 +- polar2grid/readers/aws1_mwr_l1b_nc.py | 2 +- polar2grid/readers/mirs.py | 2 +- polar2grid/resample/_resample_scene.py | 10 +++++----- polar2grid/resample/resample_decisions.py | 2 +- polar2grid/tests/test_add_coastlines.py | 2 +- .../tests/test_filters/test_day_night.py | 14 +++++++------- polar2grid/tests/test_glue.py | 6 +++--- polar2grid/utils/config.py | 1 - .../utils/convert_grids_conf_to_yaml.py | 2 +- polar2grid/writers/awips_tiled.py | 2 -- polar2grid/writers/hdf5.py | 18 +++++++++--------- pyproject.toml | 19 +++++++++++++------ swbundle/geocat2scmi.py | 2 +- 27 files changed, 71 insertions(+), 69 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 596ffd10..6ba3b4df 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -123,7 +123,7 @@ image_pathname = os.path.join(image_dst, image_fn) if os.path.isfile(image_pathname): continue - elif image_url.startswith("http://") or image_url.startswith("https://"): # or image_url.startswith('ftp://'): + elif image_url.startswith(("http://", "https://")): # or image_url.startswith('ftp://'): print("Downloading example image: {}".format(image_url)) with urllib.request.urlopen(image_url) as remote_img, open(image_pathname, "wb") as local_img: copyfileobj(remote_img, local_img) diff --git a/doc/source/toctree_filter.py b/doc/source/toctree_filter.py index b117f1a2..dedb8808 100644 --- a/doc/source/toctree_filter.py +++ b/doc/source/toctree_filter.py @@ -37,7 +37,7 @@ class TocTreeFilt(TocTree): """ option_spec = FILTER_OPTION_SPEC - hasPat = re.compile(r"^\s*:(.+):(.+)$") + has_pat = re.compile(r"^\s*:(.+):(.+)$") # Remove any entries in the content that we dont want and strip # out any filter prefixes that we want but obviously don't want the @@ -48,7 +48,7 @@ def filter_entries(self, entries): excl = self.state.document.settings.env.config.toc_filter_exclude filtered = [] for e in entries: - m = self.hasPat.match(e) + m = self.has_pat.match(e) if m is not None: if m.groups()[0] not in excl: filtered.append(m.groups()[1]) diff --git a/polar2grid/_glue_argparser.py b/polar2grid/_glue_argparser.py index eeab944c..290bdd26 100644 --- a/polar2grid/_glue_argparser.py +++ b/polar2grid/_glue_argparser.py @@ -149,7 +149,7 @@ def _separate_scene_init_load_args(self, reader_subgroups) -> None: reader_specific_args, reader_specific_load_args = self._parse_reader_args(reader_subgroups) # argparse will combine "extended" arguments like `products` automatically # and products should only be provided to the load arguments, not reader creation - for _reader_name, _reader_args in reader_specific_args.items(): + for _reader_args in reader_specific_args.values(): _reader_args.pop("products", None) # Parse provided files and search for files if provided directories @@ -527,9 +527,9 @@ def add_resample_argument_groups(parser, is_polar2grid=None): group_1 = parser.add_argument_group(title="Resampling") if is_polar2grid: - DEBUG_EWA = bool(int(os.getenv("P2G_EWA_LEGACY", "0"))) + debug_ewa = bool(int(os.getenv("P2G_EWA_LEGACY", "0"))) methods = ["ewa", "native", "nearest"] - if DEBUG_EWA: + if debug_ewa: methods.append("ewa_legacy") group_1.add_argument( @@ -627,7 +627,7 @@ def add_resample_argument_groups(parser, is_polar2grid=None): group_1.add_argument( "--grid-configs", nargs="+", - default=tuple(), + default=(), help="Specify additional grid configuration files. (.conf for legacy CSV grids, .yaml for SatPy-style areas)", ) group_1.add_argument( @@ -667,7 +667,7 @@ def add_resample_argument_groups(parser, is_polar2grid=None): "Value is in geocentric meters regardless of input or output projection. " "By default this will be estimated based on input and output projection and pixel size.", ) - return tuple([group_1]) + return (group_1,) def _retitle_optional_arguments(parser): diff --git a/polar2grid/add_coastlines.py b/polar2grid/add_coastlines.py index ca5ce8a9..a03236ce 100644 --- a/polar2grid/add_coastlines.py +++ b/polar2grid/add_coastlines.py @@ -179,9 +179,7 @@ def _add_rivers_arguments(parser: argparse.ArgumentParser) -> None: default="c", help="Resolution of rivers to add (crude, low, intermediate, high, full)", ) - group.add_argument( - "--rivers-level", choices=range(0, 11), type=int, default=5, help="Level of detail for river lines" - ) + group.add_argument("--rivers-level", choices=range(11), type=int, default=5, help="Level of detail for river lines") group.add_argument( "--rivers-outline", default=["blue"], nargs="*", help="Color of river lines (color name or 3 RGB integers)" ) @@ -326,7 +324,7 @@ def main(argv=sys.argv[1:]): parser = get_parser() args = parser.parse_args(argv) - levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG] + levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] logging.basicConfig(level=levels[min(3, args.verbosity)]) add_polar2grid_config_paths() @@ -514,7 +512,7 @@ def _process_one_image( _add_colorbar_to_image(img, colormap=cmap, **colorbar_kwargs) kwargs = {} - if output_filename.endswith(".tif") or output_filename.endswith(".tiff"): + if output_filename.endswith((".tif", ".tiff")): # Copy geotiff/tiff tags if output is TIFF geotiff_tags = {tag_num: tag_val for tag_num, tag_val in all_tiff_tags.items() if tag_num > 30000} kwargs = {"tiffinfo": geotiff_tags} diff --git a/polar2grid/compare.py b/polar2grid/compare.py index d3708d18..6db384e4 100644 --- a/polar2grid/compare.py +++ b/polar2grid/compare.py @@ -133,8 +133,8 @@ def plot_array(array1, array2, cmap="viridis", vmin=None, vmax=None, **kwargs): import matplotlib.pyplot as plt LOG.info("Plotting arrays...") - vmin = vmin if vmin else -10 - vmax = vmax if vmax else 10 + vmin = vmin or -10 + vmax = vmax or 10 fig, (ax1, ax2) = plt.subplots(2, 2) array3 = array1 - array2 @@ -765,7 +765,7 @@ def main(argv=sys.argv[1:]): parser.add_argument("input2", help="Second filename or directory to compare. This is typicall the actual output.") args = parser.parse_args(argv) - levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG] + levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] logging.basicConfig(level=levels[min(3, args.verbosity)]) compare_kwargs = { "shape": tuple(args.shape), diff --git a/polar2grid/core/containers.py b/polar2grid/core/containers.py index c985ec23..d590f7be 100644 --- a/polar2grid/core/containers.py +++ b/polar2grid/core/containers.py @@ -73,7 +73,7 @@ def __str__(self): @property def is_static(self): return all( - [self[x] is not None for x in ["height", "width", "cell_height", "cell_width", "origin_x", "origin_y"]] + self[x] is not None for x in ["height", "width", "cell_height", "cell_width", "origin_x", "origin_y"] ) @property diff --git a/polar2grid/core/dtype.py b/polar2grid/core/dtype.py index 59911405..82f2d263 100644 --- a/polar2grid/core/dtype.py +++ b/polar2grid/core/dtype.py @@ -44,7 +44,7 @@ def str_to_dtype(dtype_str: str) -> np.dtype: try: return getattr(np, dtype_str) - except AttributeError: + except AttributeError as err: raise ValueError("Not a valid data type string: %s" % (dtype_str,)) from err diff --git a/polar2grid/core/script_utils.py b/polar2grid/core/script_utils.py index befec893..95ac8d7b 100644 --- a/polar2grid/core/script_utils.py +++ b/polar2grid/core/script_utils.py @@ -257,7 +257,7 @@ def _get_group_actions(self, group): This does not group the subgroup options as their own dictionaries. """ - these_actions = [action for action in group._group_actions] + these_actions = list(group._group_actions) # get actions if this group has even more subgroups for subgroup in group._action_groups: these_actions += self._get_group_actions(subgroup) diff --git a/polar2grid/debug_data.py b/polar2grid/debug_data.py index 506e1293..444bb15c 100644 --- a/polar2grid/debug_data.py +++ b/polar2grid/debug_data.py @@ -53,7 +53,7 @@ def main(): parser.add_argument("files", nargs="*", help="Files to load data from") args = parser.parse_args() - levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG] + levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] logging.basicConfig(level=levels[min(3, args.verbosity)]) dask.config.set(num_workers=4) diff --git a/polar2grid/filters/_utils.py b/polar2grid/filters/_utils.py index 8557ca42..944abd03 100644 --- a/polar2grid/filters/_utils.py +++ b/polar2grid/filters/_utils.py @@ -26,7 +26,7 @@ import dask.array as da import numpy as np -from pyresample.boundary import AreaBoundary, AreaDefBoundary, Boundary +from pyresample.boundary import AreaBoundary, Boundary from pyresample.geometry import AreaDefinition, SwathDefinition, get_geostationary_bounding_box_in_lonlats from pyresample.spherical import SphPolygon diff --git a/polar2grid/glue.py b/polar2grid/glue.py index b560ced3..a0be5fd1 100644 --- a/polar2grid/glue.py +++ b/polar2grid/glue.py @@ -264,10 +264,10 @@ def _create_profile_html_if(create_profile: Literal[False] | None | str, project else: profile_filename = create_profile - start_time = datetime.now() + start_time = datetime.now().astimezone() with CacheProfiler() as cprof, ResourceProfiler() as rprof, Profiler() as prof: yield - end_time = datetime.now() + end_time = datetime.now().astimezone() profile_filename = profile_filename.format( project_name=project_name, @@ -424,7 +424,7 @@ def _prepare_initial_logging(arg_parser, glue_name: str) -> bool: if args.log_fn is None: rename_log = True args.log_fn = glue_name + "_fail.log" - levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG] + levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] if os.getenv("P2G_ALLOW_TRACE"): from satpy.utils import TRACE_LEVEL @@ -487,8 +487,8 @@ def _set_preferred_chunk_size(preferred_chunk_size: int) -> Iterator[None]: def _handle_missing_deps_keyerror(dep_key_error: KeyError) -> None: miss_dep_exc = getattr(dep_key_error, "__context__", None) if not isinstance(miss_dep_exc, MissingDependencies): - raise - flat_queries = sorted(set(data_query for dep_set in miss_dep_exc.missing_dependencies for data_query in dep_set)) + raise dep_key_error + flat_queries = sorted({data_query for dep_set in miss_dep_exc.missing_dependencies for data_query in dep_set}) flat_queries_dicts = [dq.to_dict() for dq in flat_queries] plural_s = "s" if len(flat_queries_dicts) > 1 else "" LOG.error( @@ -500,7 +500,7 @@ def _handle_missing_deps_keyerror(dep_key_error: KeyError) -> None: ), ) ) - LOG.debug("Unknown product requested", exc_info=True) + LOG.debug("Unknown product requested", exc_info=dep_key_error) def _persist_swath_definition_in_scene(scn: Scene) -> None: diff --git a/polar2grid/grids/config_helper.py b/polar2grid/grids/config_helper.py index faac4faf..cd4aa647 100644 --- a/polar2grid/grids/config_helper.py +++ b/polar2grid/grids/config_helper.py @@ -64,7 +64,7 @@ def determine_projection(center_lon: float, center_lat: float, proj4_str: str = "+lon_0=%(center_lon)0.5f +units=m" ) - proj4_str = proj4_str % dict(center_lon=center_lon, center_lat=center_lat) + proj4_str = proj4_str % {"center_lon": center_lon, "center_lat": center_lat} return CRS.from_user_input(proj4_str) diff --git a/polar2grid/grids/manager.py b/polar2grid/grids/manager.py index 50831564..e78c38da 100644 --- a/polar2grid/grids/manager.py +++ b/polar2grid/grids/manager.py @@ -221,7 +221,7 @@ def read_grids_config_str(config_str, convert_coords=True): def _generate_valid_parts_in_config_str(config_str: str): for line in config_str.split("\n"): # Skip comments and empty lines - if not line or line.startswith("#") or line.startswith("\n"): + if not line or line.startswith(("#", "\n")): continue # Clean up the configuration line diff --git a/polar2grid/readers/_base.py b/polar2grid/readers/_base.py index 801d16e7..9ae7a5dd 100644 --- a/polar2grid/readers/_base.py +++ b/polar2grid/readers/_base.py @@ -107,7 +107,7 @@ def get_available_products( "Provided readers are not configured in %s. All products will be listed with internal Satpy names.", self._binary_name, ) - return sorted(set([x["name"] for x in possible_satpy_ids])), [], [] + return sorted({x["name"] for x in possible_satpy_ids}), [], [] return self._alias_handler.available_product_names( p2g_product_names, available_custom_products, possible_satpy_ids ) diff --git a/polar2grid/readers/aws1_mwr_l1b_nc.py b/polar2grid/readers/aws1_mwr_l1b_nc.py index f6da4499..56178df5 100644 --- a/polar2grid/readers/aws1_mwr_l1b_nc.py +++ b/polar2grid/readers/aws1_mwr_l1b_nc.py @@ -80,7 +80,7 @@ import logging from argparse import ArgumentParser, _ArgumentGroup -from satpy import DataQuery, Scene +from satpy import DataQuery from ._base import ReaderProxyBase diff --git a/polar2grid/readers/mirs.py b/polar2grid/readers/mirs.py index 4b01f70d..b2cf3b87 100644 --- a/polar2grid/readers/mirs.py +++ b/polar2grid/readers/mirs.py @@ -241,7 +241,7 @@ def get_default_products(self) -> list[str]: """Get products to load if users hasn't specified any others.""" all_btemps = list(self._btemp_channels_from_satpy(self.scn.all_dataset_names())) defaults = ["rain_rate"] - defaults += set(all_btemps) & set(["btemp_88v", "btemp_89v1"]) + defaults += set(all_btemps) & {"btemp_88v", "btemp_89v1"} return defaults def get_all_products(self): diff --git a/polar2grid/resample/_resample_scene.py b/polar2grid/resample/_resample_scene.py index b20a5a82..cb16afe2 100644 --- a/polar2grid/resample/_resample_scene.py +++ b/polar2grid/resample/_resample_scene.py @@ -334,8 +334,8 @@ def _filter_scene_with_grid_coverage( ): if area_def is not None and resampler != "native" and coverage_threshold > 0.0 and not has_dynamic_extents: logger.info("Checking products for sufficient output grid coverage (grid: '%s')...", area_name) - filter = ResampleCoverageFilter(target_area=area_def, coverage_fraction=coverage_threshold) - scene_to_resample = filter.filter_scene(scene_to_resample) + coverage_filter = ResampleCoverageFilter(target_area=area_def, coverage_fraction=coverage_threshold) + scene_to_resample = coverage_filter.filter_scene(scene_to_resample) if scene_to_resample is None: logger.warning("No products were found to overlap with '%s' grid.", area_name) return None, None @@ -374,13 +374,13 @@ def _areas_to_resample( ) -> ListOfAreas: areas = areas_to_resample if areas is None: - if resampler in ["native"]: - logging.debug("Using default resampling target area 'MAX'.") + if resampler == "native": + logger.debug("Using default resampling target area 'MAX'.") areas = ["MAX"] elif default_target is None: raise ValueError("No destination grid/area specified and no default available (use -g flag).") else: - logging.debug("Using default resampling target area '%s'.", default_target) + logger.debug("Using default resampling target area '%s'.", default_target) areas = [default_target] elif not areas: areas = [None] diff --git a/polar2grid/resample/resample_decisions.py b/polar2grid/resample/resample_decisions.py index 6b71737d..c0ca21e7 100644 --- a/polar2grid/resample/resample_decisions.py +++ b/polar2grid/resample/resample_decisions.py @@ -77,7 +77,7 @@ def add_config_to_tree(self, *config_files): continue resampling_section = resample_config.get(self.prefix, {}) if not resampling_section: - logging.debug("Config '{}' has no '{}' section or it is empty".format(config_file, self.prefix)) + logger.debug("Config '{}' has no '{}' section or it is empty".format(config_file, self.prefix)) continue recursive_dict_update(conf, resampling_section) elif isinstance(config_file, dict): diff --git a/polar2grid/tests/test_add_coastlines.py b/polar2grid/tests/test_add_coastlines.py index 32df583e..c7648aba 100644 --- a/polar2grid/tests/test_add_coastlines.py +++ b/polar2grid/tests/test_add_coastlines.py @@ -245,7 +245,7 @@ def _check_used_colormap(passed_cmap, has_colors, include_cmap_tag, include_scal def _check_exp_image_colors(image_arr, colormap, color_idx, has_colors): exp_raw_values = list(colormap.keys()) - cmap_colors = list(set(color[color_idx] for color in colormap.values())) + cmap_colors = list({color[color_idx] for color in colormap.values()}) exp_colors = cmap_colors if has_colors else exp_raw_values r_uniques = np.unique(image_arr[:, :, color_idx]) np.testing.assert_allclose(r_uniques, exp_colors) diff --git a/polar2grid/tests/test_filters/test_day_night.py b/polar2grid/tests/test_filters/test_day_night.py index cdbb3a10..0126754a 100644 --- a/polar2grid/tests/test_filters/test_day_night.py +++ b/polar2grid/tests/test_filters/test_day_night.py @@ -44,10 +44,10 @@ def test_daynight_filter_no_filter_check_cases(filter_cls, criteria, viirs_sdr_i scn["I01"] = viirs_sdr_i01_data_array if criteria == "default": - filter = filter_cls() + dn_filter = filter_cls() else: - filter = filter_cls(criteria) - new_scn = filter.filter_scene(scn) + dn_filter = filter_cls(criteria) + new_scn = dn_filter.filter_scene(scn) assert new_scn is not None assert new_scn is not scn assert "I01" in new_scn @@ -68,8 +68,8 @@ def test_daynight_filter_filter_cases(filter_cls, add_nans, kwargs, viirs_sdr_i0 _replace_swath_def_with_some_bounding_nans(scn["I01"]) criteria = {"standard_name": ["toa_bidirectional_reflectance"]} - filter = filter_cls(criteria, **kwargs) - new_scn = filter.filter_scene(scn) + dn_filter = filter_cls(criteria, **kwargs) + new_scn = dn_filter.filter_scene(scn) assert new_scn is None # all filtered assert "I01" in scn @@ -105,8 +105,8 @@ def test_daynight_filter_basic_composite_case(viirs_sdr_i04_data_array): scn["I05"] = i05_data_arr scn.load(["ifog"]) - filter = NightCoverageFilter({"standard_name": "temperature_difference"}, night_fraction=0.2) - new_scn = filter.filter_scene(scn) + dn_filter = NightCoverageFilter({"standard_name": "temperature_difference"}, night_fraction=0.2) + new_scn = dn_filter.filter_scene(scn) assert new_scn is not None assert "ifog" not in new_scn diff --git a/polar2grid/tests/test_glue.py b/polar2grid/tests/test_glue.py index f29c0671..2db51165 100644 --- a/polar2grid/tests/test_glue.py +++ b/polar2grid/tests/test_glue.py @@ -88,7 +88,7 @@ def extra_viirs_composite_path(tmp_path_factory): tmp_path = tmp_path_factory.mktemp("extra_viirs_composite_path") _create_fake_comp(tmp_path) _create_fake_comp_enh(tmp_path) - yield [tmp_path] + return [tmp_path] def _create_fake_comp(tmp_path): @@ -178,12 +178,12 @@ def extra_viirs_enhancement_file(tmp_path_factory): viirs_enh_file = enh_path / "blahblahblah.yaml" with open(viirs_enh_file, "w") as comp_file: yaml.dump(enh_dict, comp_file) - yield [viirs_enh_file] + return [viirs_enh_file] @pytest.fixture(scope="session") def extra_viirs_comp_and_enh(extra_viirs_composite_path, extra_viirs_enhancement_file): - yield extra_viirs_composite_path + extra_viirs_enhancement_file + return extra_viirs_composite_path + extra_viirs_enhancement_file @contextlib.contextmanager diff --git a/polar2grid/utils/config.py b/polar2grid/utils/config.py index fd4afd05..64aa7d8f 100644 --- a/polar2grid/utils/config.py +++ b/polar2grid/utils/config.py @@ -26,7 +26,6 @@ import importlib.metadata as impm import importlib.resources as impr import os -import sys from collections.abc import Mapping, MutableMapping import satpy diff --git a/polar2grid/utils/convert_grids_conf_to_yaml.py b/polar2grid/utils/convert_grids_conf_to_yaml.py index 9f5f315b..2bf4b0ad 100644 --- a/polar2grid/utils/convert_grids_conf_to_yaml.py +++ b/polar2grid/utils/convert_grids_conf_to_yaml.py @@ -110,7 +110,7 @@ def _add_origin(grid_info: dict, area_dict: dict, crs: CRS, dx: float, dy: float area_dict["upper_left_extent"] = {"x": ox, "y": oy, "units": grid_info["grid_origin_units"]} -def ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds): +def ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds): # noqa: N803 (mirrors the PyYAML API) """Dump the data to YAML in ordered fashion.""" class OrderedDumper(Dumper): diff --git a/polar2grid/writers/awips_tiled.py b/polar2grid/writers/awips_tiled.py index 62ae030e..c494601a 100644 --- a/polar2grid/writers/awips_tiled.py +++ b/polar2grid/writers/awips_tiled.py @@ -100,8 +100,6 @@ def add_writer_argument_groups(parser, group=None): - import argparse - if group is None: group = parser.add_argument_group(title="AWIPS Tiled Writer") # group_1.add_argument('--file-pattern', default=DEFAULT_OUTPUT_PATTERN, diff --git a/polar2grid/writers/hdf5.py b/polar2grid/writers/hdf5.py index 0e3393da..d2cb993f 100644 --- a/polar2grid/writers/hdf5.py +++ b/polar2grid/writers/hdf5.py @@ -134,7 +134,7 @@ def iter_by_area(self, datasets: list[xr.DataArray]): return datasets_by_area.items() @staticmethod - def open_HDF5_filehandle(output_filename: str, append: bool = True): + def _open_hdf5_filehandle(output_filename: str, append: bool = True): """Open a HDF5 file handle.""" if os.path.isfile(output_filename): if append: @@ -254,7 +254,7 @@ def save_datasets( if not all_equal(output_names): LOG.warning("More than one output filename possible. Writing to only '{}'.".format(filename)) - HDF5_fh = self.open_HDF5_filehandle(filename, append=append) + hdf5_fh = self._open_hdf5_filehandle(filename, append=append) datasets_by_area = self.iter_by_area(dataset) # Initialize source/targets at start of each new AREA grouping. @@ -266,7 +266,7 @@ def save_datasets( area, data_arrs, filename, - HDF5_fh, + hdf5_fh, dtype, append, compression, @@ -288,17 +288,17 @@ def save_datasets( return targets, sources def _save_data_arrays_and_area( - self, area, data_arrs, filename, HDF5_fh, dtype, append, compression, add_geolocation + self, area, data_arrs, filename, hdf5_fh, dtype, append, compression, add_geolocation ): # open HDF5 file handle, check if group already exists. - parent_group = self.create_proj_group(filename, HDF5_fh, area) + parent_group = self.create_proj_group(filename, hdf5_fh, area) dsets = [] targets = [] if add_geolocation: chunks = data_arrs[0].chunks geo_sets, file_targets = self.write_geolocation( - HDF5_fh, filename, parent_group, area, dtype, append, compression, chunks + hdf5_fh, filename, parent_group, area, dtype, append, compression, chunks ) dsets.extend(geo_sets) targets.extend(file_targets) @@ -306,7 +306,7 @@ def _save_data_arrays_and_area( for data_arr in data_arrs: try: dask_arr, target_file = self._save_data_array( - HDF5_fh, filename, data_arr, parent_group, dtype, compression + hdf5_fh, filename, data_arr, parent_group, dtype, compression ) except ValueError: if os.path.isfile(filename): @@ -316,10 +316,10 @@ def _save_data_arrays_and_area( targets.append(target_file) return dsets, targets - def _save_data_array(self, HDF5_fh, filename, data_arr, parent_group, dtype, compression): + def _save_data_array(self, hdf5_fh, filename, data_arr, parent_group, dtype, compression): hdf_subgroup = "{}/{}".format(parent_group, data_arr.attrs.get("p2g_name", data_arr.attrs["name"])) file_var = FakeHDF5(filename, hdf_subgroup) - self.create_variable(HDF5_fh, hdf_subgroup, data_arr, dtype, compression) + self.create_variable(hdf5_fh, hdf_subgroup, data_arr, dtype, compression) return data_arr.data, file_var diff --git a/pyproject.toml b/pyproject.toml index 65e9e15d..accdef75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,15 +100,23 @@ line-length = 120 [tool.ruff.lint] # See https://docs.astral.sh/ruff/rules/ -select = ["E", "W", "B", "D", "I", "T10", "C90", "UP"] +# "G" (flake8-logging-format) is deliberately not selected: Polar2Grid logs at DEBUG to a +# file in nearly every run, so lazy %-formatting in log calls buys nothing. Use f-strings. +select = [ + "A", "B", "C4", "C90", "D", "DTZ", "E", "F", "FA", "FURB", "I", "ISC", + "LOG", "N", "NPY", "PERF", "PIE", "PLE", "PT", "Q", "T10", "UP", "W", +] # Remove D416 when all docstrings have been converted to google-style -# UP031/UP032 (%-format and .format() -> f-string) are deferred: ~73 sites, and the -# %-formatting in logging calls must stay lazy. Handle them in a separate pass. +# UP031/UP032 (%-format and .format() -> f-string) are deferred: ~73 sites. +# Handle them in a separate pass. ignore = ["D101", "D102", "D103", "D104", "D105", "D106", "D107", "E203", "D416", "UP031", "UP032"] [tool.ruff.lint.per-file-ignores] -"doc/source/conf.py" = ["E501"] -"polar2grid/readers/*.py" = ["D205", "D400", "D415", "S101"] # assert allowed in tests +# Sphinx requires the module-level name `copyright`. +"doc/source/conf.py" = ["E501", "A001"] +"polar2grid/readers/*.py" = ["D205", "D400", "D415"] +# Satpy represents observation times as naive UTC datetimes; the fixtures mirror that. +"polar2grid/tests/*" = ["DTZ001"] [tool.ruff.lint.pydocstyle] convention = "google" @@ -118,4 +126,3 @@ max-complexity = 10 [tool.ruff.lint.isort] known-first-party = ["polar2grid"] -known-third-party = ["src"] diff --git a/swbundle/geocat2scmi.py b/swbundle/geocat2scmi.py index ded63e77..248f1c3a 100644 --- a/swbundle/geocat2scmi.py +++ b/swbundle/geocat2scmi.py @@ -51,7 +51,7 @@ def main(): writer_init_args = {ga.dest: getattr(args, ga.dest) for ga in subgroups[2]._group_actions} writer_call_args = {ga.dest: getattr(args, ga.dest) for ga in subgroups[3]._group_actions} - levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG] + levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] logging.basicConfig(level=levels[min(3, args.verbosity)], filename=args.log_fn) scn = Scene(**scene_args) From d2003a3d3041bde6e8fb8c11d49d0ee13c306569 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 2 Sep 2026 15:41:22 -0500 Subject: [PATCH 2/7] More code cleanup with new 'ruff' rules --- doc/source/conf.py | 11 ++++------- polar2grid/_glue_argparser.py | 13 ++----------- polar2grid/compare.py | 10 ++-------- polar2grid/composites/enhanced.py | 2 +- polar2grid/core/script_utils.py | 9 ++++----- polar2grid/filters/_base.py | 9 ++------- polar2grid/glue.py | 2 +- polar2grid/grids/manager.py | 3 +-- polar2grid/readers/modis_l1b.py | 2 +- polar2grid/readers/nucaps.py | 2 +- polar2grid/readers/omps_edr.py | 2 +- polar2grid/readers/viirs_edr.py | 2 +- polar2grid/resample/_resample_scene.py | 9 ++++----- polar2grid/tests/test_compare.py | 16 +++++++++------- polar2grid/tests/test_filters/test_day_night.py | 5 +---- polar2grid/tests/test_glue.py | 2 +- .../tests/test_resample/test_resample_scene.py | 2 +- polar2grid/utils/convert_grids_conf_to_yaml.py | 5 +---- polar2grid/utils/search_noaa_s3_viirs.py | 2 +- polar2grid/writers/binary.py | 2 +- pyproject.toml | 12 ++++++++++-- 21 files changed, 50 insertions(+), 72 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 6ba3b4df..6e6e6f7a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -133,8 +133,8 @@ server = parts[2] ftp_fn = "/".join(parts[3:]) ftp = ftplib.FTP(server, user="ftp") # hope for anonymous - out_file = open(image_pathname, "wb") - ftp.retrbinary("RETR {}".format(ftp_fn), out_file.write) + with open(image_pathname, "wb") as out_file: + ftp.retrbinary("RETR {}".format(ftp_fn), out_file.write) else: raise ValueError("Not sure how to download image: {}".format(image_url)) @@ -176,10 +176,7 @@ def setup(app): ] # API docs -if is_geo2grid: - _apidoc_excluded_paths = ["../../polar2grid/readers/amsr2_l1b.py"] -else: - _apidoc_excluded_paths = [] +_apidoc_excluded_paths = ["../../polar2grid/readers/amsr2_l1b.py"] if is_geo2grid else [] apidoc_separate_modules = True apidoc_include_private = True @@ -595,7 +592,7 @@ def setup(app): if area_title is None: continue proj = area_dict["projection"] - crs = CRS.from_user_input(proj["EPSG"] if "EPSG" in proj else proj) + crs = CRS.from_user_input(proj.get("EPSG", proj)) title_underline = "^" * len(area_title) rst_str = f""" .. _grid_{area_name}: diff --git a/polar2grid/_glue_argparser.py b/polar2grid/_glue_argparser.py index 290bdd26..3dcab8ce 100644 --- a/polar2grid/_glue_argparser.py +++ b/polar2grid/_glue_argparser.py @@ -485,18 +485,9 @@ def float_or_false(val): def _supported_writers(is_polar2grid: bool = False) -> list[str]: + writers = ["geotiff", "awips_tiled"] if is_polar2grid: - writers = [ - "geotiff", - "awips_tiled", - "binary", - "hdf5", - ] - else: - writers = [ - "geotiff", - "awips_tiled", - ] + writers += ["binary", "hdf5"] return writers diff --git a/polar2grid/compare.py b/polar2grid/compare.py index 6db384e4..81b5db12 100644 --- a/polar2grid/compare.py +++ b/polar2grid/compare.py @@ -84,10 +84,7 @@ class FileComparisonResults: def any_failed(self) -> bool: if self.files_missing or self.unknown_file_type: return True - for sub_result in self.sub_results: - if sub_result.failed: - return True - return False + return any(sub_result.failed for sub_result in self.sub_results) def isclose_array(array1, array2, atol=0.0, rtol=0.0, margin_of_error=0.0, **kwargs) -> ArrayComparisonResult: @@ -677,10 +674,7 @@ def _generate_matplotlib_thumbnail(input_arr, output_thumbnail_path, max_width=5 def _get_mpl_figsize(input_shape, max_width) -> tuple[int, int]: # dpi 100 => 51 fig_width = max_width / 100.0 - if len(input_shape) == 1: - fig_height = fig_width - else: - fig_height = input_shape[0] * (fig_width / input_shape[1]) + fig_height = fig_width if len(input_shape) == 1 else input_shape[0] * (fig_width / input_shape[1]) return fig_width, fig_height diff --git a/polar2grid/composites/enhanced.py b/polar2grid/composites/enhanced.py index 3dd15cbd..0ca65d9d 100644 --- a/polar2grid/composites/enhanced.py +++ b/polar2grid/composites/enhanced.py @@ -45,7 +45,7 @@ def __call__(self, projectables, nonprojectables=None, **attrs): data = enhance2dataset(data) data.attrs = new_attrs if isinstance(data.attrs.get("sensor"), set) and len(data.attrs["sensor"]) == 1: - data.attrs["sensor"] = list(data.attrs["sensor"])[0] + data.attrs["sensor"] = next(iter(data.attrs["sensor"])) return super().__call__([data], **attrs) diff --git a/polar2grid/core/script_utils.py b/polar2grid/core/script_utils.py index 95ac8d7b..c6449c88 100644 --- a/polar2grid/core/script_utils.py +++ b/polar2grid/core/script_utils.py @@ -149,9 +149,8 @@ def rename_log_file(new_filename): # move the old file if os.path.isfile(new_filename): - with open(new_filename, "a") as new_file: - with open(fn) as old_file: - new_file.write(old_file.read()) + with open(new_filename, "a") as new_file, open(fn) as old_file: + new_file.write(old_file.read()) os.remove(fn) else: os.rename(fn, new_filename) @@ -279,8 +278,8 @@ def parse_args(self, *args, **kwargs): args.subgroup_args = defaultdict(dict) for subgroup_title in subgroup_titles: try: - subgroup = [x for x in self._action_groups if x.title == subgroup_title][0] - except IndexError: + subgroup = next(x for x in self._action_groups if x.title == subgroup_title) + except StopIteration: # we don't have any loggers configured at this point print("WARNING: Couldn't find argument group '%s' in configured parser" % (subgroup_title,)) continue diff --git a/polar2grid/filters/_base.py b/polar2grid/filters/_base.py index ecda35a4..7e5bd42d 100644 --- a/polar2grid/filters/_base.py +++ b/polar2grid/filters/_base.py @@ -60,10 +60,7 @@ def _matches_criteria(self, data_arr: DataArray): # if no criteria was provided then no products will be checked return False - for filter_key, filter_list in self._filter_criteria.items(): - if attrs.get(filter_key) in filter_list: - return True - return False + return any(attrs.get(filter_key) in filter_list for filter_key, filter_list in self._filter_criteria.items()) def _filter_data_array(self, data_arr: DataArray, _cache: dict): """Check if this DataArray should be removed. @@ -74,10 +71,8 @@ def _filter_data_array(self, data_arr: DataArray, _cache: dict): meaning it should be kept. """ - if not self._matches_criteria(data_arr): - return False # Subclasses should implement further logic here - return True + return self._matches_criteria(data_arr) def filter_scene(self, scene: Scene): """Create a new Scene with filtered DataArrays removed.""" diff --git a/polar2grid/glue.py b/polar2grid/glue.py index a0be5fd1..8d683643 100644 --- a/polar2grid/glue.py +++ b/polar2grid/glue.py @@ -253,7 +253,7 @@ def _get_glue_name(args): @contextlib.contextmanager -def _create_profile_html_if(create_profile: Literal[False] | None | str, project_name: str, glue_name: str): +def _create_profile_html_if(create_profile: Literal[False] | str | None, project_name: str, glue_name: str): from dask.diagnostics import CacheProfiler, Profiler, ResourceProfiler, visualize if create_profile is False: diff --git a/polar2grid/grids/manager.py b/polar2grid/grids/manager.py index e78c38da..df389bb0 100644 --- a/polar2grid/grids/manager.py +++ b/polar2grid/grids/manager.py @@ -254,9 +254,8 @@ def read_grids_config(config_filepath, convert_coords=True): class GridManager: """Object that holds grid information about the grids added to it.""" - grid_information: dict[str, dict] = {} - def __init__(self, *grid_configs): + self.grid_information: dict[str, dict] = {} for grid_config in grid_configs: LOG.debug("Loading grid configuration '%s'", grid_config) self.add_grid_config(grid_config) diff --git a/polar2grid/readers/modis_l1b.py b/polar2grid/readers/modis_l1b.py index 97d9aabb..4af0b8b7 100644 --- a/polar2grid/readers/modis_l1b.py +++ b/polar2grid/readers/modis_l1b.py @@ -29,7 +29,7 @@ reader can be specified to the ``polar2grid.sh`` script by using the reader name ``modis`` or ``modis_l1b``. -This reader’s default remapping algorithm is ewa for Elliptical Weighted +This reader's default remapping algorithm is ewa for Elliptical Weighted Averaging resampling. The ``--weight-delta-max`` parameter set to 10 and the ``--weight-distance-max`` parameter set to 1. diff --git a/polar2grid/readers/nucaps.py b/polar2grid/readers/nucaps.py index 6e7da39c..5dc562db 100644 --- a/polar2grid/readers/nucaps.py +++ b/polar2grid/readers/nucaps.py @@ -291,7 +291,7 @@ def get_default_products(self) -> list[str]: def get_all_products(self) -> list[str]: """Get all polar2grid products that could be loaded.""" - reader = self.scn._readers["nucaps"] # noqa + reader = self.scn._readers["nucaps"] products = ["Topography", "Surface_Pressure", "Skin_Temperature"] for base_name in PRESSURE_BASED: products.extend(reader.pressure_dataset_names[base_name]) diff --git a/polar2grid/readers/omps_edr.py b/polar2grid/readers/omps_edr.py index 139a5e8b..e5d29506 100644 --- a/polar2grid/readers/omps_edr.py +++ b/polar2grid/readers/omps_edr.py @@ -159,7 +159,7 @@ def add_reader_argument_groups( return group, None -def _int_or_none(value: str) -> None | int: +def _int_or_none(value: str) -> int | None: if value.lower() == "none": return None return int(value) diff --git a/polar2grid/readers/viirs_edr.py b/polar2grid/readers/viirs_edr.py index 5b9b8614..2af57385 100644 --- a/polar2grid/readers/viirs_edr.py +++ b/polar2grid/readers/viirs_edr.py @@ -207,7 +207,7 @@ def add_reader_argument_groups( return group, None -def _int_or_none(value: str) -> None | int: +def _int_or_none(value: str) -> int | None: if value.lower() == "none": return None return int(value) diff --git a/polar2grid/resample/_resample_scene.py b/polar2grid/resample/_resample_scene.py index cb16afe2..9f868f4d 100644 --- a/polar2grid/resample/_resample_scene.py +++ b/polar2grid/resample/_resample_scene.py @@ -230,7 +230,8 @@ def _get_groups_to_resample( ) -> dict: resampling_dtree = ResamplerDecisionTree.from_configs() resampling_groups = {} - for data_id in input_scene.keys(): + # NOTE: iterating a Scene yields DataArrays, not DataIDs -- .keys() is required here. + for data_id in input_scene.keys(): # noqa: SIM118 resampling_args = resampling_dtree.find_match(**input_scene[data_id].attrs) default_resampler = resampling_args.get("resampler") resampler_kwargs = resampling_args.get("kwargs", {}).copy() @@ -246,10 +247,8 @@ def _get_groups_to_resample( def _default_grid(resampler, is_polar2grid): if resampler in [None, "native"]: - default_target = "MAX" - else: - default_target = "wgs84_fit" if is_polar2grid else "MAX" - return default_target + return "MAX" + return "wgs84_fit" if is_polar2grid else "MAX" def _hashable_kwargs(kwargs): diff --git a/polar2grid/tests/test_compare.py b/polar2grid/tests/test_compare.py index ea285b33..90344715 100644 --- a/polar2grid/tests/test_compare.py +++ b/polar2grid/tests/test_compare.py @@ -53,8 +53,9 @@ def _create_geotiffs(base_dir, img_data): for idx, img_arr in enumerate(img_data): band_count = 1 if img_arr.ndim == 2 else img_arr.shape[0] gtiff_fn = os.path.join(base_dir, f"test{idx}.tif") - with ignore_no_georef(): - with rasterio.open( + with ( + ignore_no_georef(), + rasterio.open( gtiff_fn, "w", driver="GTiff", @@ -62,11 +63,12 @@ def _create_geotiffs(base_dir, img_data): height=img_arr.shape[-2], width=img_arr.shape[-1], dtype=img_arr.dtype, - ) as gtiff_file: - if img_arr.ndim == 2: - img_arr = img_arr[None, :, :] - for band_idx, band_arr in enumerate(img_arr): - gtiff_file.write(band_arr, band_idx + 1) + ) as gtiff_file, + ): + if img_arr.ndim == 2: + img_arr = img_arr[None, :, :] + for band_idx, band_arr in enumerate(img_arr): + gtiff_file.write(band_arr, band_idx + 1) def _create_hdf5(base_dir, img_data): diff --git a/polar2grid/tests/test_filters/test_day_night.py b/polar2grid/tests/test_filters/test_day_night.py index 0126754a..4aa73d79 100644 --- a/polar2grid/tests/test_filters/test_day_night.py +++ b/polar2grid/tests/test_filters/test_day_night.py @@ -43,10 +43,7 @@ def test_daynight_filter_no_filter_check_cases(filter_cls, criteria, viirs_sdr_i scn = Scene() scn["I01"] = viirs_sdr_i01_data_array - if criteria == "default": - dn_filter = filter_cls() - else: - dn_filter = filter_cls(criteria) + dn_filter = filter_cls() if criteria == "default" else filter_cls(criteria) new_scn = dn_filter.filter_scene(scn) assert new_scn is not None assert new_scn is not scn diff --git a/polar2grid/tests/test_glue.py b/polar2grid/tests/test_glue.py index 2db51165..88f7ae2a 100644 --- a/polar2grid/tests/test_glue.py +++ b/polar2grid/tests/test_glue.py @@ -357,7 +357,7 @@ def test_extra_config_path( if str(extra_cpath).endswith(".yaml"): # the YAML was copied to a temp directory, check for that extra_cpath = gettempdir() - path_idx = captured.err.index(f"Adding enhancement configuration from file: {str(extra_cpath)}") + path_idx = captured.err.index(f"Adding enhancement configuration from file: {extra_cpath!s}") assert builtin_path_idx < path_idx def test_avhrr_list_products(self, avhrr_l1b_1_scene, chtmpdir, capsys): diff --git a/polar2grid/tests/test_resample/test_resample_scene.py b/polar2grid/tests/test_resample/test_resample_scene.py index 5c43f89f..31461684 100644 --- a/polar2grid/tests/test_resample/test_resample_scene.py +++ b/polar2grid/tests/test_resample/test_resample_scene.py @@ -261,5 +261,5 @@ def test_partial_filter(viirs_sdr_i01_data_array): grid_coverage=0.05, ) assert len(scenes_to_save) == 1 - new_scn, data_ids = scenes_to_save[0] + new_scn, _data_ids = scenes_to_save[0] assert len(new_scn.keys()) == 1 # I01 diff --git a/polar2grid/utils/convert_grids_conf_to_yaml.py b/polar2grid/utils/convert_grids_conf_to_yaml.py index 2bf4b0ad..944d335e 100644 --- a/polar2grid/utils/convert_grids_conf_to_yaml.py +++ b/polar2grid/utils/convert_grids_conf_to_yaml.py @@ -65,10 +65,7 @@ def grid_info_to_yaml_dict(grid_info: dict) -> dict: def crs_to_proj_dict(crs: CRS) -> dict: warnings.filterwarnings("ignore", module="pyproj", category=UserWarning) - if crs.to_epsg() is not None: - proj_dict = {"EPSG": crs.to_epsg()} - else: - proj_dict = crs.to_dict() + proj_dict = {"EPSG": crs.to_epsg()} if crs.to_epsg() is not None else crs.to_dict() _remove_unnecessary_proj_params(proj_dict) return proj_dict diff --git a/polar2grid/utils/search_noaa_s3_viirs.py b/polar2grid/utils/search_noaa_s3_viirs.py index 000a75b5..ed0326bc 100644 --- a/polar2grid/utils/search_noaa_s3_viirs.py +++ b/polar2grid/utils/search_noaa_s3_viirs.py @@ -70,7 +70,7 @@ def band_to_prefix(band: str) -> str: "band" GITCO is converted to "VIIRS-IMG-GEO-TC". """ - if band.startswith("G"): + if band.startswith("G"): # noqa: SIM108 (nesting these ternaries is less readable) # M-band and I-band geolocation are only available for terrain-corrected (TC) band_type = "GEO" if band[1] == "D" else "GEO-TC" else: diff --git a/polar2grid/writers/binary.py b/polar2grid/writers/binary.py index e706685b..1e523f16 100644 --- a/polar2grid/writers/binary.py +++ b/polar2grid/writers/binary.py @@ -122,7 +122,7 @@ def _prep_data(self, data: xr.DataArray, dtype: np.dtype, fill_value) -> da.Arra final_data = final_data * (rmax - rmin) + rmin final_data = clip_to_data_type(final_data, dtype) - same_fill = np.isnan(fill) and np.isnan(fill_value) or fill == fill_value + same_fill = (np.isnan(fill) and np.isnan(fill_value)) or fill == fill_value if data.dtype == dtype and same_fill: return final_data diff --git a/pyproject.toml b/pyproject.toml index accdef75..bced2473 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,12 +104,20 @@ line-length = 120 # file in nearly every run, so lazy %-formatting in log calls buys nothing. Use f-strings. select = [ "A", "B", "C4", "C90", "D", "DTZ", "E", "F", "FA", "FURB", "I", "ISC", - "LOG", "N", "NPY", "PERF", "PIE", "PLE", "PT", "Q", "T10", "UP", "W", + "LOG", "N", "NPY", "PERF", "PIE", "PLE", "PT", "Q", "RUF", "SIM", + "T10", "UP", "W", ] # Remove D416 when all docstrings have been converted to google-style # UP031/UP032 (%-format and .format() -> f-string) are deferred: ~73 sites. # Handle them in a separate pass. -ignore = ["D101", "D102", "D103", "D104", "D105", "D106", "D107", "E203", "D416", "UP031", "UP032"] +ignore = [ + "D101", "D102", "D103", "D104", "D105", "D106", "D107", "D416", "E203", + "UP031", "UP032", + # RUF005: `[*a, b]` over `a + [b]` is a readability preference, ~22 sites. + "RUF005", + # RUF013: implicit Optional (`x: str = None`); 7 sites, deferred to a typing pass. + "RUF013", +] [tool.ruff.lint.per-file-ignores] # Sphinx requires the module-level name `copyright`. From 47966c4508b5ef2f6406add0f950758908db21a7 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 2 Sep 2026 16:10:55 -0500 Subject: [PATCH 3/7] Fix implicit optional type annotations --- polar2grid/compare.py | 2 +- polar2grid/filters/_base.py | 2 +- polar2grid/filters/day_night.py | 12 +++++++++--- polar2grid/filters/resample_coverage.py | 6 +++--- polar2grid/grids/config_helper.py | 2 +- pyproject.toml | 2 -- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/polar2grid/compare.py b/polar2grid/compare.py index 81b5db12..8aef0d4b 100644 --- a/polar2grid/compare.py +++ b/polar2grid/compare.py @@ -304,7 +304,7 @@ def compare_image(im1_name, im2_name, atol=0.0, margin_of_error=0.0, **kwargs) - def _get_image_array( - img_filename: str, variable: str = None, shape: tuple | None = None, dtype: np.dtype | None = None + img_filename: str, variable: str | None = None, shape: tuple | None = None, dtype: np.dtype | None = None ) -> np.ndarray | None: from PIL import Image diff --git a/polar2grid/filters/_base.py b/polar2grid/filters/_base.py index 7e5bd42d..e601d828 100644 --- a/polar2grid/filters/_base.py +++ b/polar2grid/filters/_base.py @@ -47,7 +47,7 @@ class BaseFilter: FILTER_MSG = "Unloading '{}' due to filtering." - def __init__(self, product_filter_criteria: dict = None): + def __init__(self, product_filter_criteria: dict | bool | None = None): """Initialize thresholds and default search criteria.""" self._filter_criteria = product_filter_criteria diff --git a/polar2grid/filters/day_night.py b/polar2grid/filters/day_night.py index de35db76..ce696a4f 100644 --- a/polar2grid/filters/day_night.py +++ b/polar2grid/filters/day_night.py @@ -101,7 +101,9 @@ class SunlightCoverageFilter(BaseFilter): FILTER_MSG = "Unloading '{}' because there is not enough day/night coverage." - def __init__(self, product_filter_criteria: dict = None, sza_threshold: float = 100.0, fraction: float = 0.1): + def __init__( + self, product_filter_criteria: dict | None = None, sza_threshold: float = 100.0, fraction: float = 0.1 + ): """Initialize thresholds and default search criteria.""" super().__init__(product_filter_criteria) self._sza_threshold = sza_threshold @@ -139,7 +141,9 @@ class DayCoverageFilter(SunlightCoverageFilter): FILTER_MSG = "Unloading '{}' because there is not enough day data." - def __init__(self, product_filter_criteria: dict = None, sza_threshold: float = 100.0, day_fraction: float = 0.1): + def __init__( + self, product_filter_criteria: dict | None = None, sza_threshold: float = 100.0, day_fraction: float = 0.1 + ): """Initialize thresholds and default search criteria.""" super().__init__(product_filter_criteria, sza_threshold=sza_threshold, fraction=day_fraction) @@ -152,7 +156,9 @@ class NightCoverageFilter(SunlightCoverageFilter): FILTER_MSG = "Unloading '{}' because there is not enough night data." - def __init__(self, product_filter_criteria: dict = None, sza_threshold: float = 100.0, night_fraction: float = 0.1): + def __init__( + self, product_filter_criteria: dict | None = None, sza_threshold: float = 100.0, night_fraction: float = 0.1 + ): """Initialize thresholds and default search criteria.""" super().__init__(product_filter_criteria, sza_threshold=sza_threshold, fraction=night_fraction) diff --git a/polar2grid/filters/resample_coverage.py b/polar2grid/filters/resample_coverage.py index d4197204..a752c582 100644 --- a/polar2grid/filters/resample_coverage.py +++ b/polar2grid/filters/resample_coverage.py @@ -56,9 +56,9 @@ class ResampleCoverageFilter(BaseFilter): def __init__( self, - product_filter_criteria: dict = True, - target_area: PRGeometry = None, - coverage_fraction: float = None, + product_filter_criteria: dict | bool = True, + target_area: PRGeometry | None = None, + coverage_fraction: float | None = None, ): """Initialize thresholds and default search criteria.""" super().__init__(product_filter_criteria) diff --git a/polar2grid/grids/config_helper.py b/polar2grid/grids/config_helper.py index cd4aa647..94bcd16b 100644 --- a/polar2grid/grids/config_helper.py +++ b/polar2grid/grids/config_helper.py @@ -44,7 +44,7 @@ ) -def determine_projection(center_lon: float, center_lat: float, proj4_str: str = None) -> CRS: +def determine_projection(center_lon: float, center_lat: float, proj4_str: str | None = None) -> CRS: """Get the 'best' projection to be used based on the center longitude and latitude provided.""" abs_lat = abs(center_lat) if proj4_str is None: diff --git a/pyproject.toml b/pyproject.toml index bced2473..d8469f6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,8 +115,6 @@ ignore = [ "UP031", "UP032", # RUF005: `[*a, b]` over `a + [b]` is a readability preference, ~22 sites. "RUF005", - # RUF013: implicit Optional (`x: str = None`); 7 sites, deferred to a typing pass. - "RUF013", ] [tool.ruff.lint.per-file-ignores] From 04cb7edd86bff9369d9dfa7a69cdb01a0da9c03d Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 2 Sep 2026 16:20:10 -0500 Subject: [PATCH 4/7] Switch to absolute imports --- polar2grid/filters/_filter_scene.py | 3 ++- polar2grid/readers/abi_l2_nc.py | 3 ++- polar2grid/readers/acspo.py | 3 ++- polar2grid/readers/avhrr_l1b_aapp.py | 3 ++- polar2grid/readers/mersi2_l1b.py | 3 ++- polar2grid/readers/omps_edr.py | 3 ++- polar2grid/resample/_resample_scene.py | 2 +- polar2grid/resample/resample_decisions.py | 6 ++++-- polar2grid/tests/test_configs.py | 3 ++- polar2grid/tests/test_filters/test_utils.py | 3 +-- polar2grid/tests/test_main.py | 3 ++- polar2grid/utils/search_noaa_s3_viirs.py | 2 +- pyproject.toml | 15 ++++++++++----- 13 files changed, 33 insertions(+), 19 deletions(-) diff --git a/polar2grid/filters/_filter_scene.py b/polar2grid/filters/_filter_scene.py index 8cffd460..9ee983de 100644 --- a/polar2grid/filters/_filter_scene.py +++ b/polar2grid/filters/_filter_scene.py @@ -25,7 +25,8 @@ from satpy import Scene -from ..utils.dynamic_imports import get_reader_attr +from polar2grid.utils.dynamic_imports import get_reader_attr + from .day_night import DayCoverageFilter, NightCoverageFilter logger = logging.getLogger(__name__) diff --git a/polar2grid/readers/abi_l2_nc.py b/polar2grid/readers/abi_l2_nc.py index 330c300e..ed24bb17 100644 --- a/polar2grid/readers/abi_l2_nc.py +++ b/polar2grid/readers/abi_l2_nc.py @@ -74,7 +74,8 @@ from argparse import ArgumentParser, _ArgumentGroup -from ..core.script_utils import BooleanFilterAction +from polar2grid.core.script_utils import BooleanFilterAction + from ._base import ReaderProxyBase PREFERRED_CHUNK_SIZE: int = 1356 diff --git a/polar2grid/readers/acspo.py b/polar2grid/readers/acspo.py index 5342203e..4f0b46be 100644 --- a/polar2grid/readers/acspo.py +++ b/polar2grid/readers/acspo.py @@ -44,7 +44,8 @@ from satpy import DataQuery -from ..core.script_utils import BooleanFilterAction +from polar2grid.core.script_utils import BooleanFilterAction + from ._base import ReaderProxyBase DEFAULT_PRODUCTS = ["sst"] diff --git a/polar2grid/readers/avhrr_l1b_aapp.py b/polar2grid/readers/avhrr_l1b_aapp.py index 4ed291ad..bfa6991d 100644 --- a/polar2grid/readers/avhrr_l1b_aapp.py +++ b/polar2grid/readers/avhrr_l1b_aapp.py @@ -56,7 +56,8 @@ from satpy import DataQuery, Scene -from ..core.script_utils import ExtendConstAction +from polar2grid.core.script_utils import ExtendConstAction + from ._base import ReaderProxyBase logger = logging.getLogger(__name__) diff --git a/polar2grid/readers/mersi2_l1b.py b/polar2grid/readers/mersi2_l1b.py index 3ac18851..6c14c21d 100644 --- a/polar2grid/readers/mersi2_l1b.py +++ b/polar2grid/readers/mersi2_l1b.py @@ -101,7 +101,8 @@ from satpy import DataQuery, Scene -from ..core.script_utils import ExtendConstAction +from polar2grid.core.script_utils import ExtendConstAction + from ._base import ReaderProxyBase logger = logging.getLogger(__name__) diff --git a/polar2grid/readers/omps_edr.py b/polar2grid/readers/omps_edr.py index e5d29506..667fbd39 100644 --- a/polar2grid/readers/omps_edr.py +++ b/polar2grid/readers/omps_edr.py @@ -78,7 +78,8 @@ from satpy import DataQuery -from ..core.script_utils import BooleanFilterAction +from polar2grid.core.script_utils import BooleanFilterAction + from ._base import ReaderProxyBase PREFERRED_CHUNK_SIZE: int = 6400 diff --git a/polar2grid/resample/_resample_scene.py b/polar2grid/resample/_resample_scene.py index 9f868f4d..9d900ea2 100644 --- a/polar2grid/resample/_resample_scene.py +++ b/polar2grid/resample/_resample_scene.py @@ -33,10 +33,10 @@ from satpy import Scene from satpy.area import get_area_def +from polar2grid.filters._utils import PRGeometry from polar2grid.filters.resample_coverage import ResampleCoverageFilter from polar2grid.grids import GridManager -from ..filters._utils import PRGeometry from .resample_decisions import ResamplerDecisionTree logger = logging.getLogger(__name__) diff --git a/polar2grid/resample/resample_decisions.py b/polar2grid/resample/resample_decisions.py index c0ca21e7..f5bbc96d 100644 --- a/polar2grid/resample/resample_decisions.py +++ b/polar2grid/resample/resample_decisions.py @@ -71,7 +71,9 @@ def add_config_to_tree(self, *config_files): for config_file in config_files: if os.path.isfile(config_file): with open(config_file) as fd: - resample_config = yaml.load(fd, Loader=yaml.UnsafeLoader) + # These are config files shipped with the package or named by the + # user via --extra-config-path, not untrusted input. + resample_config = yaml.load(fd, Loader=yaml.UnsafeLoader) # noqa: S506 if resample_config is None: # empty file continue @@ -84,7 +86,7 @@ def add_config_to_tree(self, *config_files): recursive_dict_update(conf, config_file) else: logger.debug("Loading resampling config string") - d = yaml.load(config_file, Loader=yaml.UnsafeLoader) + d = yaml.load(config_file, Loader=yaml.UnsafeLoader) # noqa: S506 (trusted local config) if not isinstance(d, dict): raise ValueError("YAML file doesn't exist or string is not YAML dict: {}".format(config_file)) recursive_dict_update(conf, d) diff --git a/polar2grid/tests/test_configs.py b/polar2grid/tests/test_configs.py index 15aa7acd..4370f162 100644 --- a/polar2grid/tests/test_configs.py +++ b/polar2grid/tests/test_configs.py @@ -44,4 +44,5 @@ def pytest_generate_tests(metafunc): def test_valid_yaml_files(yaml_config_file): """Test basic YAML syntax for all config files.""" with open(yaml_config_file) as yaml_file: - yaml.load(yaml_file, Loader=yaml.UnsafeLoader) + # Satpy enhancement YAML uses `!!python/name:` tags, which SafeLoader rejects. + yaml.load(yaml_file, Loader=yaml.UnsafeLoader) # noqa: S506 diff --git a/polar2grid/tests/test_filters/test_utils.py b/polar2grid/tests/test_filters/test_utils.py index 2db97e90..9477164e 100644 --- a/polar2grid/tests/test_filters/test_utils.py +++ b/polar2grid/tests/test_filters/test_utils.py @@ -31,8 +31,7 @@ from satpy.tests.utils import CustomScheduler from polar2grid.filters._utils import boundary_for_area - -from .._fixture_utils import generate_lonlat_data +from polar2grid.tests._fixture_utils import generate_lonlat_data def _swath_def_nan_rows() -> SwathDefinition: diff --git a/polar2grid/tests/test_main.py b/polar2grid/tests/test_main.py index d57dc85f..5de240b7 100644 --- a/polar2grid/tests/test_main.py +++ b/polar2grid/tests/test_main.py @@ -43,5 +43,6 @@ def test_main_call(main_func, exp_reader, capsys): def test_call_main_script(): import subprocess + import sys - subprocess.check_call(["python3", "-m", "polar2grid", "-h"]) + subprocess.check_call([sys.executable, "-m", "polar2grid", "-h"]) diff --git a/polar2grid/utils/search_noaa_s3_viirs.py b/polar2grid/utils/search_noaa_s3_viirs.py index ed0326bc..5da9cb41 100644 --- a/polar2grid/utils/search_noaa_s3_viirs.py +++ b/polar2grid/utils/search_noaa_s3_viirs.py @@ -44,7 +44,7 @@ DEFAULT_BANDS = VALID_BANDS # Special token accepted by --band that expands to every band -ALL_BANDS_TOKEN = "ALL" +ALL_BANDS_TOKEN = "ALL" # noqa: S105 (a CLI keyword, not a credential) def parse_band(value: str) -> list[str]: diff --git a/pyproject.toml b/pyproject.toml index d8469f6b..4e7b1ac1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,8 +104,8 @@ line-length = 120 # file in nearly every run, so lazy %-formatting in log calls buys nothing. Use f-strings. select = [ "A", "B", "C4", "C90", "D", "DTZ", "E", "F", "FA", "FURB", "I", "ISC", - "LOG", "N", "NPY", "PERF", "PIE", "PLE", "PT", "Q", "RUF", "SIM", - "T10", "UP", "W", + "LOG", "N", "NPY", "PERF", "PIE", "PLE", "PT", "Q", "RUF", "S", "SIM", + "T10", "TID", "UP", "W", ] # Remove D416 when all docstrings have been converted to google-style # UP031/UP032 (%-format and .format() -> f-string) are deferred: ~73 sites. @@ -118,11 +118,16 @@ ignore = [ ] [tool.ruff.lint.per-file-ignores] -# Sphinx requires the module-level name `copyright`. -"doc/source/conf.py" = ["E501", "A001"] +# Sphinx requires the module-level name `copyright`. The example-image downloader checks +# the URL scheme itself (S310) and supports FTP servers on purpose (S321). +"doc/source/conf.py" = ["E501", "A001", "S310", "S321"] "polar2grid/readers/*.py" = ["D205", "D400", "D415"] # Satpy represents observation times as naive UTC datetimes; the fixtures mirror that. -"polar2grid/tests/*" = ["DTZ001"] +# S101: `assert` is how tests assert. +"polar2grid/tests/*" = ["DTZ001", "S101"] +# behave harness: `assert` is the assertion mechanism, and the steps deliberately shell +# out to the built software bundle. +"integration_tests/*" = ["S101", "S602"] [tool.ruff.lint.pydocstyle] convention = "google" From c17190384a5122c1fa6cd6627fc7541e8101c3a2 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 2 Sep 2026 21:22:13 -0500 Subject: [PATCH 5/7] Split website CI into two separate jobs This should make them each run on a separate runner and avoid throttling limits for accessing remote resources. This commit also splits the example image downloads by project. --- .github/workflows/ci.yaml | 17 +++++++-------- doc/source/conf.py | 44 +++++++++++---------------------------- 2 files changed, 19 insertions(+), 42 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7c0c3f98..a13afc4e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,8 +13,12 @@ env: jobs: # build website website: - name: Build Documentation + name: Build Documentation (${{ matrix.p2g_doc }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + p2g_doc: ["polar", "geo"] steps: - name: Checkout source uses: actions/checkout@v7 @@ -53,19 +57,12 @@ jobs: run: | pip install --no-deps -e . - - name: Build Polar2Grid Website - shell: bash -l {0} - run: | - cd doc - make clean - make html SPHINXOPTS="-W --keep-going" - - - name: Build Geo2Grid Website + - name: Build Website shell: bash -l {0} run: | cd doc make clean - make html POLAR2GRID_DOC="geo" SPHINXOPTS="-W --keep-going" + make html POLAR2GRID_DOC="${{ matrix.p2g_doc }}" SPHINXOPTS="-W --keep-going" test: name: Tests diff --git a/doc/source/conf.py b/doc/source/conf.py index 6e6e6f7a..e56cf79b 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -38,27 +38,17 @@ # Handle building documentation for polar2grid or geo2grid is_geo2grid = "geo" in os.getenv("POLAR2GRID_DOC", "polar").lower() -# Hack to download example images instead of storing them in git -images = ( - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/amsr2/images_basic/gcom-w1_amsr2_btemp_36.5h_20160719_190300_wgs84_fit.jpg", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/amsr2/images_nrl/gcom-w1_amsr2_btemp_89.0ah_20160719_190300_lcc_fit.jpg", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/amsr2/images_nrl/gcom-w1_amsr2_btemp_89.0ah_20160719_190300_lcc_fit.basic_overlay_example.png", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/amsr2/images_nrl/gcom-w1_amsr2_btemp_89.0ah_20160719_190300_lcc_fit.advanced_overlay.png", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/viirs/overlay/npp_viirs_true_color_20170305_193251_lcc_fit_overlay.png", +# Hack to download example images instead of storing them in git. +# Split per project so each documentation build only downloads the images its own +# pages render; no image is currently shared between the two projects. +_POLAR2GRID_IMAGES = ( "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_3_0_examples/viirs/images_basic/noaa20_viirs_true_color_20220919_175331_lcc_fit.jpg", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_3_0_examples/viirs/overlay/noaa20_viirs_true_color_20220919_175331_lcc_fit_overlay.png", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/viirs/dnb/VIIRS_DNB_Enhancement_Comparison.png", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_3_0_examples/viirs/images_basic/noaa21_viirs_true_color_20260223_182634_miami.png", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/modis/images_basic/terra_modis_false_color_20170319_163000_miami.jpg", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_3_0_examples/modis/images_basic/terra_modis_false_color_20170319_163000_miami_p2g_v3.png", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/modis/awips/SSEC_AWIPS_aqua_modis_bt28_211e_20170308_181800.nc.png", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/modis/awips/SSEC_AWIPS_aqua_modis_vis02_211e_20170308_181800.nc.png", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/amsr2/images_nrl/gcom-w1_amsr2_btemp_89.0ah_20160719_190300_lcc_fit_color.jpg", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/viirs/merge/VIIRS_False_Color_Side_by_Side_Example_P2G.png", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/viirs/overlay/VIIRS_true_color_in_google_earth.jpg", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_3_0_examples/viirs/images_basic/noaa20_viirs_true_color_20220919_175331_lcc_fit_in_google_earth.jpg", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/viirs/merge/my_false_color.jpg", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/modis/awips/modis_true_color_awips_20170308_1818.png", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/viirs/dnb/HNCC_DNB_Band_Example.png", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/modis/awips/modis_true_color_example.png", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/modis/awips/modis_vis02_example.png", @@ -76,9 +66,6 @@ "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_3_0_examples/asci/VIIRS_TCF_example_with_overlays.png", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_3_0_examples/asci/VIIRS_NDVI_example_with_overlays.png", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_3_0_examples/asci/VIIRS_SurfReflectance_True_Color_example_with_overlays.png", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/acspo/npp_viirs_sst_20191216_072134_acspo_sst_rescaled_wcolor.png", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/acspo/npp_viirs_sst_20191216_072134_acspo_sst_rescaled_wcolor_colortable_resize.png", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/acspo/npp_viirs_sst_20191216_072134_acspo_sst_final_resize.png", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_3_0_examples/amsr2/gcom-w1_amsr2_btemp_89.0ah_20220910_233500_wgs84_fit.png", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_3_0_examples/amsr2/gcom-w1_amsr2_btemp_89.0ah_20220910_233500_wgs84_lcc_fit_rescaled_and_colorized.png", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_3_0_examples/amsr2/GAASP_Rain_Rate_example_with_overlays.png", @@ -86,34 +73,26 @@ "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_3_0_examples/viirs/aws/JPSS_AWS_VIIRS_Single_Granule_True_Color_Example.png", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_3_0_examples/viirs/aws/NOAA21_NA_orbit_track_2026_02_08.gif", "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_3_0_examples/viirs/aws/NOAA21_VIIRS_Aggregated_AWS_example.png", - "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/ahi/HIMAWARI-8_AHI_true_color_20181112_233020_perth_example.png", - "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/ahi/HIMAWARI-8_AHI_true_color_20181112_233020_perth.png", + # Only used by readers/viirs_edr_flood.rst, which is excluded from both builds. + # Left here pending review rather than deleted: + # "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/flood/Flood_Legend.png", +) +_GEO2GRID_IMAGES = ( "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/ahi/HIMAWARI-9_AHI_true_color_20221128_233000_perth.png", "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/ahi/HIMAWARI-9_AHI_true_color_20221128_233000_perth_example.png", - "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/abi_20181219_1745_montage.png", - "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/abi_20181219_1745_montage.jpg", "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/abi_20260303_1830_montage.png", - "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/GOES-16_ABI_RadC_natural_color_20181219_174215_GOES-East.png", "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/GOES-18_ABI_RadF_natural_color_20221115_183020_GOES-West_cutout.png", - "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/GOES-16_ABI_RadC_true_color_20190104_195718_GOES-East.png", "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/GOES-19_ABI_RadM1_true_color_20251028_150027_GOES-East.png", - "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/GOES-16_ABI_RadF_true_color_night_20181112_123034_GOES-East.jpg", - "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/GOES-16_ABI_RadF_true_color_night_20181112_123034_GOES-East_new.png", "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/GOES-19_ABI_RadF_true_color_night_20260303_123020_GOES-East.png", - "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/GOES-16_ABI_RadF_true_color_night_microphysics_20181112_123034_GOES-East_newenhancement.png", - "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/GOES-16_ABI_RadF_true_color_night_microphysics_20181112_123034_GOES-East.jpg", - "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/GOES-16_ABI_RadF_true_color_night_microphysics_20181112_123034_GOES-East_new.png", "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/GOES-19_ABI_RadF_true_color_night_microphysics_20260303_123020_GOES-East.png", - "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/my_goes16_abi_naturalcolor.png", "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi/my_goes18_abi_naturalcolor.png", "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi_l2/GOES-17_ABI_TEMP_20221123_183117_GOES-West_original.png", "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi_l2/goes17_overlay_true_color_cloud_temperature.png", - "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi_l2/my_goes17_abi_ctt.png", "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi_l2/my_goes17_abi_ctt1.png", "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi_l2/GOES-16_ABI_AOD_20240429_225020_GOES-East.png", "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi_l2/GOES-16_ABI_AOD_20240429_225020_GOES-East_woverlays.png", - "https://bin.ssec.wisc.edu/pub/CSPP/p2g_v_2_1_examples/flood/Flood_Legend.png", ) +images = _GEO2GRID_IMAGES if is_geo2grid else _POLAR2GRID_IMAGES script_path = os.path.dirname(os.path.realpath(__file__)) image_dst = os.path.join(script_path, "_static", "example_images") os.makedirs(image_dst, exist_ok=True) @@ -543,7 +522,6 @@ def setup(app): intersphinx_mapping = { "python": ("https://docs.python.org/3", None), "numpy": ("https://numpy.org/doc/stable", None), - "scipy": ("https://docs.scipy.org/doc/scipy", None), "xarray": ("https://docs.xarray.dev/en/stable", None), "dask": ("https://docs.dask.org/en/latest", None), "pyresample": ("https://pyresample.readthedocs.io/en/stable", None), @@ -551,6 +529,8 @@ def setup(app): "satpy": ("https://satpy.readthedocs.io/en/stable", None), "trollimage": ("https://trollimage.readthedocs.io/en/stable", None), } +# Don't let an unreachable inventory hang the build for the OS TCP timeout. +intersphinx_timeout = 10 # Generate builtin grids list From 3161a794082922a4fed1f13cf03fa3c77e68d220 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 3 Sep 2026 06:45:20 -0500 Subject: [PATCH 6/7] Move sphinx in-conf.py checks to separate extension modules --- .claude/skills/add-reader/SKILL.md | 2 +- .claude/skills/build-docs/SKILL.md | 25 +++-- AGENTS.md | 14 ++- doc/source/_ext/__init__.py | 1 + doc/source/_ext/config_checks.py | 75 ++++++++++++++ doc/source/{ => _ext}/doi_role.py | 0 doc/source/_ext/example_images.py | 63 ++++++++++++ doc/source/_ext/grids_list.py | 100 ++++++++++++++++++ doc/source/{ => _ext}/toctree_filter.py | 0 doc/source/conf.py | 131 +++--------------------- doc/source/dev_guide/adding_readers.rst | 2 +- pyproject.toml | 9 +- 12 files changed, 283 insertions(+), 139 deletions(-) create mode 100644 doc/source/_ext/__init__.py create mode 100644 doc/source/_ext/config_checks.py rename doc/source/{ => _ext}/doi_role.py (100%) create mode 100644 doc/source/_ext/example_images.py create mode 100644 doc/source/_ext/grids_list.py rename doc/source/{ => _ext}/toctree_filter.py (100%) diff --git a/.claude/skills/add-reader/SKILL.md b/.claude/skills/add-reader/SKILL.md index 9760baa6..eb51f737 100644 --- a/.claude/skills/add-reader/SKILL.md +++ b/.claude/skills/add-reader/SKILL.md @@ -170,7 +170,7 @@ section. `:geo2grid:` prefix inside the `toctree-filt` block. If the reader serves **both** projects, either drop the prefix (unprefixed entries are always -included, see `doc/source/toctree_filter.py`) or list it once per prefix — the repo does the latter +included, see `doc/source/_ext/toctree_filter.py`) or list it once per prefix — the repo does the latter for `geotiff` in `doc/source/writers/index.rst`. Either way skip step 7 entirely. Note that setting both `is_polar2grid_reader` and `is_geo2grid_reader` does not by itself make a reader dual-project in the docs — the toctree entry does. diff --git a/.claude/skills/build-docs/SKILL.md b/.claude/skills/build-docs/SKILL.md index 77138c15..48f76ba8 100644 --- a/.claude/skills/build-docs/SKILL.md +++ b/.claude/skills/build-docs/SKILL.md @@ -50,12 +50,12 @@ Three mechanisms select content: name or script name in a shared page. Available: `|project|`, `|script|` (`polar2grid.sh` / `geo2grid.sh`), `|script_literal|`, `|project_env|` (`$POLAR2GRID_HOME` / `$GEO2GRID_HOME`), `|cspp_abbr|`, `|cspp_title|`, plus `|ssec|`, `|cspp|`, `|viirs|`. -2. **`toctree-filt`** (`doc/source/toctree_filter.py`). Prefix entries with `:polar2grid:` or +2. **`toctree-filt`** (`doc/source/_ext/toctree_filter.py`). Prefix entries with `:polar2grid:` or `:geo2grid:`; `toc_filter_exclude` (`conf.py:252`) drops the other project's. It also accepts `:excludebuilder: latex`. A page shared by both projects gets no prefix, or is listed once per prefix. 3. **`exclude_patterns`** (`conf.py:266`). A hand-maintained per-project list of files dropped - from the build entirely, held in the `_GEO2GRID_EXCLUDES` and `_POLAR2GRID_EXCLUDES` constants. + from the build entirely, held in the `geo2grid_excludes` and `polar2grid_excludes` values. Inline conditionals use `.. ifconfig:: is_geo2grid`. ## Adding a page @@ -74,20 +74,23 @@ WARNING: document isn't included in any toctree That means the file is present in a build where nothing references it — add it to that project's `exclude_patterns`. -The reverse mistake is caught eagerly: a check at the bottom of `conf.py` validates **both** -projects' lists and raises `RuntimeError` before the build starts if either names a file that does -not exist. So deleting or renaming a page means updating the list in the same commit. +The reverse mistake is caught eagerly: `check_excluded_pages_exist()` in +`doc/source/_ext/config_checks.py` validates **both** projects' lists on `config-inited` and +raises `ConfigError` before the build starts if either names a file that does not exist. So +deleting or renaming a page means updating the list in the same commit. ## Generated versus hand-maintained Generated at build time, gitignored, **do not edit**: -- `doc/source/grids_list.rst` — built from `polar2grid/grids/grids.yaml` by `conf.py`. A new - built-in grid also needs an entry in the `grid_titles` dict in `conf.py`, or it is silently - omitted. +- `doc/source/grids_list.rst` — built from `polar2grid/grids/grids.yaml` by + `doc/source/_ext/grids_list.py`. A new built-in grid also needs an entry in that module's + `GRID_TITLES` dict, or it is silently omitted. - `doc/source/dev_guide/api/` — `sphinx.ext.apidoc` output (`apidoc_modules` in `conf.py`). -- `doc/source/_static/example_images/` — ~90 images downloaded from `bin.ssec.wisc.edu` at build - time. A new documentation image means adding a URL to the tuple near the top of `conf.py`. +- `doc/source/_static/example_images/` — images downloaded from `bin.ssec.wisc.edu` at build time + by `doc/source/_ext/example_images.py`. A new documentation image means adding a URL to + `_POLAR2GRID_IMAGES` or `_GEO2GRID_IMAGES` near the top of `conf.py`; `check_example_images_used()` + in `_ext/config_checks.py` fails the build if a listed image is unused or a used one is unlisted. Generated from Python at build time (edit the Python, not the `.rst`): reader and writer pages use `.. automodule::` for the module docstring and the `sphinxarg.ext` `.. argparse::` directive for @@ -114,6 +117,6 @@ diverged by several releases and overwrote `summary_table.rst` in place). Edit t - `conf.py`'s `version` / `release` are the **bundle** versions (Polar2Grid `3.2`, Geo2Grid `1.3`), tracking `NEWS.rst` / `NEWS_GEO2GRID.rst`. They are meant to differ from the `polar2grid` package version in `pyproject.toml` — do not "sync" them. -- `doc/source/doi_role.py` provides a `:doi:` role. +- `doc/source/_ext/doi_role.py` provides a `:doi:` role. - PDF output is built with `make latexpdf`; `latex_documents`, `latex_logo`, and `latex_appendices` are also project-conditional in `conf.py`. diff --git a/AGENTS.md b/AGENTS.md index 72d4e3df..cbdf921a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -366,12 +366,18 @@ One source tree builds both projects (`make html` vs `make html POLAR2GRID_DOC=g * Use the `rst_epilog` substitutions from `doc/source/conf.py` — `|project|`, `|script|`, `|script_literal|`, `|project_env|`, `|cspp_abbr|`, `|cspp_title|`. * Use the `toctree-filt` directive with `:polar2grid:` / `:geo2grid:` entry prefixes - (`doc/source/toctree_filter.py`), and `.. ifconfig:: is_geo2grid` for inline conditionals. + (`doc/source/_ext/toctree_filter.py`), and `.. ifconfig:: is_geo2grid` for inline conditionals. * A new `.rst` must be added to a toctree **and** to the *other* project's `exclude_patterns` in `conf.py`, or the CI `-W` build fails with "document isn't included in any toctree". The two - project lists are the `_GEO2GRID_EXCLUDES` / `_POLAR2GRID_EXCLUDES` constants; a check at the - bottom of `conf.py` raises if either one names a file that does not exist, so a page that is - removed must also be removed from the list. + project lists are the `geo2grid_excludes` / `polar2grid_excludes` configuration values; a + `config-inited` check in `doc/source/_ext/config_checks.py` raises if either one names a file + that does not exist, so a page that is removed must also be removed from the list. + +`doc/source/_ext/` is a package of local Sphinx extensions loaded by the `extensions` list in +`conf.py`. Alongside `doi_role` and `toctree_filter` it holds the code that used to sit inline in +`conf.py`: `example_images` (downloads the example images), `grids_list` (generates +`grids_list.rst`), and `config_checks` (the two `config-inited` consistency checks). Keep +`conf.py` to configuration and data; put logic here. Do not hand-edit `doc/source/grids_list.rst`, `doc/source/dev_guide/api/`, or `doc/source/_static/example_images/` — all three are generated at build time and gitignored. diff --git a/doc/source/_ext/__init__.py b/doc/source/_ext/__init__.py new file mode 100644 index 00000000..857f8bd1 --- /dev/null +++ b/doc/source/_ext/__init__.py @@ -0,0 +1 @@ +"""Local Sphinx extensions for the Polar2Grid and Geo2Grid documentation builds.""" diff --git a/doc/source/_ext/config_checks.py b/doc/source/_ext/config_checks.py new file mode 100644 index 00000000..8af72e17 --- /dev/null +++ b/doc/source/_ext/config_checks.py @@ -0,0 +1,75 @@ +"""Consistency checks for the documentation configuration. + +One source tree builds both the Polar2Grid and the Geo2Grid site, and which pages belong +to which project is expressed by hand in ``conf.py``. These checks run on ``config-inited`` +and fail the build when those hand-maintained lists drift away from the files on disk. + +Sphinx already errors on a page that references a missing image, so the half of +:func:`check_example_images_used` that earns its keep is the other direction: entries that +no built page uses. The example image list carried 29 such leftovers before it was split +per project. +""" + +import fnmatch +import glob +import os +import re +from pathlib import Path + +from sphinx.errors import ConfigError + +#: Captures the file name from a ``_static/example_images/`` reference. +_EXAMPLE_IMAGE_RE = re.compile(r"_static/example_images/([A-Za-z0-9_.\-]+)") + +#: Characters that make an ``exclude_patterns`` entry a glob rather than a literal path. +_GLOB_CHARS = "*?[" + + +def check_excluded_pages_exist(app, config): + """Fail if a literal ``exclude_patterns`` entry names a file that does not exist. + + Both projects' lists are checked, not just the one being built, so a page that is + renamed or removed is caught by whichever site builds first. + """ + srcdir = Path(app.srcdir) + all_excludes = set(config.exclude_patterns) | set(config.geo2grid_excludes) | set(config.polar2grid_excludes) + missing = sorted( + pattern + for pattern in all_excludes + if not any(glob_char in pattern for glob_char in _GLOB_CHARS) and not (srcdir / pattern).exists() + ) + if missing: + raise ConfigError("'exclude_patterns' names files that do not exist: " + ", ".join(missing)) + + +def check_example_images_used(app, config): + """Fail if this project's example image list and the pages being built disagree.""" + referenced = _referenced_example_images(Path(app.srcdir), config.exclude_patterns) + listed = {os.path.basename(image_url) for image_url in config.example_images} + if unused := sorted(listed - referenced): + raise ConfigError("example image list has entries no built page uses: " + ", ".join(unused)) + if unlisted := sorted(referenced - listed): + raise ConfigError("example image list is missing images used by pages: " + ", ".join(unlisted)) + + +def _referenced_example_images(srcdir, exclude_patterns): + """Collect the example image file names referenced by the pages this build includes.""" + referenced = set() + for rel_path in glob.glob("**/*.rst", root_dir=srcdir, recursive=True): + if any(fnmatch.fnmatch(rel_path, pattern) for pattern in exclude_patterns): + continue + with open(srcdir / rel_path) as rst_file: + referenced.update(_EXAMPLE_IMAGE_RE.findall(rst_file.read())) + return referenced + + +def setup(app): + """Declare the per-project page lists and connect the checks.""" + # Declares the `example_images` configuration value that check_example_images_used + # reads. Its downloader deliberately runs at a later priority than these checks. + app.setup_extension("_ext.example_images") + app.add_config_value("polar2grid_excludes", [], "env") + app.add_config_value("geo2grid_excludes", [], "env") + app.connect("config-inited", check_excluded_pages_exist) + app.connect("config-inited", check_example_images_used) + return {"version": "1.0.0", "parallel_read_safe": True, "parallel_write_safe": True} diff --git a/doc/source/doi_role.py b/doc/source/_ext/doi_role.py similarity index 100% rename from doc/source/doi_role.py rename to doc/source/_ext/doi_role.py diff --git a/doc/source/_ext/example_images.py b/doc/source/_ext/example_images.py new file mode 100644 index 00000000..ccfc6936 --- /dev/null +++ b/doc/source/_ext/example_images.py @@ -0,0 +1,63 @@ +"""Download the example images that the documentation pages reference. + +The images are large and are deliberately not stored in git. The list of URLs for the +project being built is the ``example_images`` configuration value, set by ``conf.py``; +this extension fetches whatever is not already on disk into ``_static/example_images``. + +See :mod:`_ext.config_checks` for the check that keeps that list and the pages that +reference the images in agreement. +""" + +import ftplib +import os +import urllib.request +from pathlib import Path +from shutil import copyfileobj + +from sphinx.errors import ConfigError +from sphinx.util import logging + +logger = logging.getLogger(__name__) + +#: Run after the default priority, so `_ext.config_checks` gets to reject a stale image +#: list before anything is fetched for it. +DOWNLOAD_PRIORITY = 800 + + +def download_example_images(app, config): + """Download every example image for this project that is not already on disk.""" + image_dst = Path(app.srcdir) / "_static" / "example_images" + image_dst.mkdir(parents=True, exist_ok=True) + + for image_url in config.example_images: + image_pathname = image_dst / os.path.basename(image_url) + if image_pathname.is_file(): + continue + logger.info("Downloading example image: %s", image_url) + if image_url.startswith(("http://", "https://")): + _download_http(image_url, image_pathname) + elif image_url.startswith("ftp://"): + _download_ftp(image_url, image_pathname) + else: + raise ConfigError(f"Not sure how to download image: {image_url}") + + +def _download_http(image_url, image_pathname): + with urllib.request.urlopen(image_url) as remote_img, open(image_pathname, "wb") as local_img: + copyfileobj(remote_img, local_img) + + +def _download_ftp(image_url, image_pathname): + parts = image_url.split("/") + server = parts[2] + ftp_fn = "/".join(parts[3:]) + ftp = ftplib.FTP(server, user="ftp") # hope for anonymous + with open(image_pathname, "wb") as out_file: + ftp.retrbinary(f"RETR {ftp_fn}", out_file.write) + + +def setup(app): + """Declare the example image list and connect the downloader.""" + app.add_config_value("example_images", (), "env") + app.connect("config-inited", download_example_images, priority=DOWNLOAD_PRIORITY) + return {"version": "1.0.0", "parallel_read_safe": True, "parallel_write_safe": True} diff --git a/doc/source/_ext/grids_list.py b/doc/source/_ext/grids_list.py new file mode 100644 index 00000000..c30261b0 --- /dev/null +++ b/doc/source/_ext/grids_list.py @@ -0,0 +1,100 @@ +"""Generate ``grids_list.rst`` from Polar2Grid's built-in grid definitions. + +The page is written into the source directory before the build reads any sources and is +gitignored. It is listed in ``exclude_patterns`` because ``grids.rst`` pulls it in with an +``include`` directive rather than a toctree entry, which is why this runs at a lower +priority than the checks in :mod:`_ext.config_checks` -- those verify that every literal +exclude entry names a file that exists. +""" + +import warnings +from pathlib import Path + +import yaml +from pyproj import CRS + +import polar2grid + +#: Run ahead of the default priority so the page exists before anything looks for it. +GENERATE_PRIORITY = 100 + +#: The grids to document, and the title to give each one. A grid that is missing here is +#: left out of the generated page. +GRID_TITLES = { + "wgs84_fit": "WGS84 Dynamic Fit", + "wgs84_fit_250": "WGS84 Dynamic Fit 250m", + "lcc_fit": "Lambert Conic Conformal Dynamic Fit", + "lcc_fit_hr": "High Resolution Lambert Conic Conformal Dynamic Fit", + "lcc_sa": "Lambert Conic Conformal - South America Centered", + "lcc_eu": "Lambert Conic Conformal - Europe Centered", + "lcc_south_africa": "Lambert Conic Conformal - South Africa Centered", + "lcc_aus": "Lambert Conic Conformal - Australia Centered", + "lcc_asia": "Lambert Conic Conformal - Asia Centered", + "polar_north_pacific": "Polar-Stereographic North Pacific", + "polar_south_pacific": "Polar-Stereographic South Pacific", + "polar_alaska": "Polar-Stereographic Alaska", + "polar_canada": "Polar-Stereographic Canada", + "polar_russia": "Polar-Stereographic Russia", + "eqc_fit": "Equirectangular Fit", + "goes_east_1km": "GOES-East 1km", + "goes_east_4km": "GOES-East 4km", + "goes_east_8km": "GOES-East 8km", + "goes_east_10km": "GOES-East 10km", + "goes_west_1km": "GOES-West 1km", + "goes_west_4km": "GOES-West 4km", + "goes_west_8km": "GOES-West 8km", + "goes_west_10km": "GOES-West 10km", +} + + +def write_grids_list(app, config): + """Write ``grids_list.rst`` describing each documented built-in grid.""" + builtin_areas_filename = Path(polar2grid.__file__).parent / "grids" / "grids.yaml" + with open(builtin_areas_filename) as yaml_file: + areas_dict = yaml.load(yaml_file, Loader=yaml.SafeLoader) + + with warnings.catch_warnings(), open(Path(app.srcdir) / "grids_list.rst", "w") as grids_list_file: + warnings.filterwarnings("ignore", module="pyproj", category=UserWarning) + for area_name, area_dict in areas_dict.items(): + area_title = GRID_TITLES.get(area_name) + if area_title is None: + continue + grids_list_file.write(_grid_rst(area_name, area_title, area_dict)) + + +def _grid_rst(area_name, area_title, area_dict): + """Render the reStructuredText section describing a single grid.""" + proj = area_dict["projection"] + crs = CRS.from_user_input(proj.get("EPSG", proj)) + title_underline = "^" * len(area_title) + rst_str = f""" +.. _grid_{area_name}: + +{area_title} +{title_underline} + +:Grid Name: {area_name} +:Description: {area_dict["description"]} +:Projection: {crs.to_string()} +""" + + if "resolution" in area_dict: + res = area_dict["resolution"] + xres = res["dx"] + yres = res["dy"] + def_units = "degrees" if crs.is_geographic else "meters" + units = res.get("units", def_units) + if xres != yres: + rst_str += f":Resolution (X): {xres} {units}\n" + rst_str += f":Resolution (Y): {yres} {units}\n" + else: + rst_str += f":Resolution: {xres} {units}\n" + if "area_extent" in area_dict: + rst_str += f":Extent: {area_dict['area_extent']}\n" + return rst_str + + +def setup(app): + """Connect the generator early enough that the page exists for the rest of the build.""" + app.connect("config-inited", write_grids_list, priority=GENERATE_PRIORITY) + return {"version": "1.0.0", "parallel_read_safe": True, "parallel_write_safe": True} diff --git a/doc/source/toctree_filter.py b/doc/source/_ext/toctree_filter.py similarity index 100% rename from doc/source/toctree_filter.py rename to doc/source/_ext/toctree_filter.py diff --git a/doc/source/conf.py b/doc/source/conf.py index e56cf79b..94abb46f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -11,16 +11,9 @@ # serve to show the default. """Polar2Grid project's sphinx documentation configuration.""" -import ftplib import os import sys -import urllib.request -import warnings from datetime import UTC, datetime -from shutil import copyfileobj - -import yaml -from pyproj import CRS # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -92,30 +85,8 @@ "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi_l2/GOES-16_ABI_AOD_20240429_225020_GOES-East.png", "https://bin.ssec.wisc.edu/pub/CSPP/g2g_examples/abi_l2/GOES-16_ABI_AOD_20240429_225020_GOES-East_woverlays.png", ) -images = _GEO2GRID_IMAGES if is_geo2grid else _POLAR2GRID_IMAGES -script_path = os.path.dirname(os.path.realpath(__file__)) -image_dst = os.path.join(script_path, "_static", "example_images") -os.makedirs(image_dst, exist_ok=True) - -for image_url in images: - image_fn = os.path.basename(image_url) - image_pathname = os.path.join(image_dst, image_fn) - if os.path.isfile(image_pathname): - continue - elif image_url.startswith(("http://", "https://")): # or image_url.startswith('ftp://'): - print("Downloading example image: {}".format(image_url)) - with urllib.request.urlopen(image_url) as remote_img, open(image_pathname, "wb") as local_img: - copyfileobj(remote_img, local_img) - elif image_url.startswith("ftp://"): - print("Downloading example image: {}".format(image_url)) - parts = image_url.split("/") - server = parts[2] - ftp_fn = "/".join(parts[3:]) - ftp = ftplib.FTP(server, user="ftp") # hope for anonymous - with open(image_pathname, "wb") as out_file: - ftp.retrbinary("RETR {}".format(ftp_fn), out_file.write) - else: - raise ValueError("Not sure how to download image: {}".format(image_url)) +# Downloaded by the `_ext.example_images` extension. +example_images = _GEO2GRID_IMAGES if is_geo2grid else _POLAR2GRID_IMAGES # -- Customize setup ----------------------------------------------------------- @@ -149,9 +120,12 @@ def setup(app): "sphinx.ext.ifconfig", "sphinx.ext.viewcode", "sphinxarg.ext", - "doi_role", - "toctree_filter", "sphinx.ext.apidoc", + "_ext.doi_role", + "_ext.toctree_filter", + "_ext.example_images", + "_ext.grids_list", + "_ext.config_checks", ] # API docs @@ -239,9 +213,9 @@ def setup(app): # today_fmt = '%B %d, %Y' # Pages that belong to only one of the two projects and are dropped from the other -# project's build. Kept as module-level constants so the consistency check at the -# bottom of this file can validate both lists no matter which project is built. -_GEO2GRID_EXCLUDES = [ +# project's build. Both are Sphinx configuration values so that `_ext.config_checks` +# can validate both lists no matter which project is being built. +geo2grid_excludes = [ "NEWS.rst", "examples/acspo_example.rst", "examples/amsr2_example.rst", @@ -274,7 +248,7 @@ def setup(app): "writers/hdf5.rst", "misc_recipes.rst", ] -_POLAR2GRID_EXCLUDES = [ +polar2grid_excludes = [ "NEWS_GEO2GRID.rst", "compositors.rst", "data_access.rst", @@ -301,7 +275,7 @@ def setup(app): "readers/viirs_edr_flood.rst", # not advertised "grids_list.rst", # included directly in grids.rst ] -exclude_patterns.extend(_GEO2GRID_EXCLUDES if is_geo2grid else _POLAR2GRID_EXCLUDES) +exclude_patterns.extend(geo2grid_excludes if is_geo2grid else polar2grid_excludes) # The reST default role (used for this markup: `text`) to use for all documents. # default_role = None @@ -531,84 +505,3 @@ def setup(app): } # Don't let an unreachable inventory hang the build for the OS TCP timeout. intersphinx_timeout = 10 - -# Generate builtin grids list - -grid_titles = { - "wgs84_fit": "WGS84 Dynamic Fit", - "wgs84_fit_250": "WGS84 Dynamic Fit 250m", - "lcc_fit": "Lambert Conic Conformal Dynamic Fit", - "lcc_fit_hr": "High Resolution Lambert Conic Conformal Dynamic Fit", - "lcc_sa": "Lambert Conic Conformal - South America Centered", - "lcc_eu": "Lambert Conic Conformal - Europe Centered", - "lcc_south_africa": "Lambert Conic Conformal - South Africa Centered", - "lcc_aus": "Lambert Conic Conformal - Australia Centered", - "lcc_asia": "Lambert Conic Conformal - Asia Centered", - "polar_north_pacific": "Polar-Stereographic North Pacific", - "polar_south_pacific": "Polar-Stereographic South Pacific", - "polar_alaska": "Polar-Stereographic Alaska", - "polar_canada": "Polar-Stereographic Canada", - "polar_russia": "Polar-Stereographic Russia", - "eqc_fit": "Equirectangular Fit", - "goes_east_1km": "GOES-East 1km", - "goes_east_4km": "GOES-East 4km", - "goes_east_8km": "GOES-East 8km", - "goes_east_10km": "GOES-East 10km", - "goes_west_1km": "GOES-West 1km", - "goes_west_4km": "GOES-West 4km", - "goes_west_8km": "GOES-West 8km", - "goes_west_10km": "GOES-West 10km", -} - -warnings.filterwarnings("ignore", module="pyproj", category=UserWarning) -builtin_areas_filename = os.path.join(script_path, "..", "..", "polar2grid", "grids", "grids.yaml") -with open(builtin_areas_filename) as yaml_file: - areas_dict = yaml.load(yaml_file, Loader=yaml.SafeLoader) -# builtin_areas = parse_area_file(os.path.join(script_path, "..", "..", "polar2grid", "grids", "grids.yaml")) -grids_list_filename = os.path.join(script_path, "grids_list.rst") -with open(grids_list_filename, "w") as grids_list_file: - for area_name, area_dict in areas_dict.items(): - area_title = grid_titles.get(area_name) - if area_title is None: - continue - proj = area_dict["projection"] - crs = CRS.from_user_input(proj.get("EPSG", proj)) - title_underline = "^" * len(area_title) - rst_str = f""" -.. _grid_{area_name}: - -{area_title} -{title_underline} - -:Grid Name: {area_name} -:Description: {area_dict["description"]} -:Projection: {crs.to_string()} -""" - - if "resolution" in area_dict: - res = area_dict["resolution"] - xres = res["dx"] - yres = res["dy"] - def_units = "degrees" if crs.is_geographic else "meters" - units = res.get("units", def_units) - if xres != yres: - rst_str += f":Resolution (X): {xres} {units}\n" - rst_str += f":Resolution (Y): {yres} {units}\n" - else: - rst_str += f":Resolution: {xres} {units}\n" - if "area_extent" in area_dict: - rst_str += f":Extent: {area_dict['area_extent']}\n" - grids_list_file.write(rst_str) - -# Guard against `exclude_patterns` rotting: every literal (non-glob) entry must name a -# file that actually exists. Both projects' lists are checked, not just the one being -# built. This must stay at the end of the file -- `grids_list.rst` is excluded but is -# generated just above. -_all_excludes = set(exclude_patterns) | set(_GEO2GRID_EXCLUDES) | set(_POLAR2GRID_EXCLUDES) -_missing_excludes = sorted( - pattern - for pattern in _all_excludes - if not any(glob_char in pattern for glob_char in "*?[") and not os.path.exists(os.path.join(script_path, pattern)) -) -if _missing_excludes: - raise RuntimeError("conf.py 'exclude_patterns' names files that do not exist: " + ", ".join(_missing_excludes)) diff --git a/doc/source/dev_guide/adding_readers.rst b/doc/source/dev_guide/adding_readers.rst index 8440192d..a4c42ce7 100644 --- a/doc/source/dev_guide/adding_readers.rst +++ b/doc/source/dev_guide/adding_readers.rst @@ -184,7 +184,7 @@ entry with no prefix is included in both. If the reader is specific to one project, the new document must also be added to the *other* project's ``exclude_patterns`` list in ``doc/source/conf.py`` -(``_POLAR2GRID_EXCLUDES`` or ``_GEO2GRID_EXCLUDES``). Both projects' +(``polar2grid_excludes`` or ``geo2grid_excludes``). Both projects' documentation is built in continuous integration with warnings treated as errors, so a document that is in neither a table of contents nor the exclude list fails the build with "document isn't included in any toctree". diff --git a/pyproject.toml b/pyproject.toml index 4e7b1ac1..e6d4b4e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,9 +118,12 @@ ignore = [ ] [tool.ruff.lint.per-file-ignores] -# Sphinx requires the module-level name `copyright`. The example-image downloader checks -# the URL scheme itself (S310) and supports FTP servers on purpose (S321). -"doc/source/conf.py" = ["E501", "A001", "S310", "S321"] +# E501: the example image URLs are longer than the line limit. +# A001: Sphinx requires the module-level name `copyright`. +"doc/source/conf.py" = ["E501", "A001"] +# The example-image downloader checks the URL scheme itself (S310) and supports FTP +# servers on purpose (S321). +"doc/source/_ext/example_images.py" = ["S310", "S321"] "polar2grid/readers/*.py" = ["D205", "D400", "D415"] # Satpy represents observation times as naive UTC datetimes; the fixtures mirror that. # S101: `assert` is how tests assert. From 4c99af664933317b889b294393329b29f6e085f0 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 3 Sep 2026 09:15:32 -0500 Subject: [PATCH 7/7] Fix old path insert logic in sphinx conf.py --- doc/source/conf.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 94abb46f..9afc773f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -15,18 +15,12 @@ import sys from datetime import UTC, datetime -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) -print("Adding the following directories to PYTHONPATH:") -BASE_PATH = "../../../" -for dirname in [ - x for x in os.listdir(BASE_PATH) if os.path.isdir(os.path.join(BASE_PATH, x)) and x.startswith("polar2grid") -]: - print("\t ", os.path.realpath(os.path.join(BASE_PATH, dirname))) - sys.path.insert(0, os.path.abspath(os.path.join(BASE_PATH, dirname))) -sys.path.append(os.path.abspath(os.path.dirname(__file__))) +# Make the polar2grid package (and this directory's ``_ext`` extensions) importable +# no matter what directory sphinx-build was run from. +DOC_SOURCE_PATH = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT_PATH = os.path.dirname(os.path.dirname(DOC_SOURCE_PATH)) +sys.path.insert(0, REPO_ROOT_PATH) +sys.path.append(DOC_SOURCE_PATH) # Handle building documentation for polar2grid or geo2grid is_geo2grid = "geo" in os.getenv("POLAR2GRID_DOC", "polar").lower()