From 3968d171cde8e06c4e5c92e52359d575fa898027 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Fri, 17 Jul 2026 12:39:32 +0100 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20telescope=20runs=20calendar=20(issu?= =?UTF-8?q?e=20#37)=20=E2=80=94=20code=20and=20templates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed from issue37-telescope-runs-calendar (809 commits) to split the GSD .planning/ documentation out of PR #41 for reviewable diff size. See the issue37-planning-docs branch for the accompanying planning artifacts. Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 10 +- .pre-commit-config.yaml | 2 +- CLAUDE.md | 446 ++++ docs/conf.py | 11 + docs/design/GSD_Claude_notes.md | 196 ++ docs/design/design.rst | 12 + docs/design/eso_feasibility_spike.rst | 174 ++ docs/design/gsd_experiment.rst | 137 + docs/design/telescope_runs_calendar.rst | 347 +++ .../tom_calendar_vs_yse_pz_calendar.rst | 175 ++ docs/design/uncertain_scheduling_spike.rst | 111 + docs/notebooks/ESO_How_to_download_data.ipynb | 592 +++++ .../pre_executed/fixtures/campaign_sample.csv | 9 + .../import_campaign_csv_demo.ipynb | 578 +++++ .../load_telescope_runs_demo.ipynb | 917 +++++++ ...ync_gemini_observation_calendar_demo.ipynb | 739 ++++++ .../sync_lco_observation_calendar_demo.ipynb | 1980 +++++++++++++++ .../pre_executed/telescope_runs_demo.ipynb | 371 +++ manage.py | 1 + pyproject.toml | 5 +- solsys_code/admin.py | 35 +- solsys_code/apps.py | 16 + solsys_code/calendar_urls.py | 23 + solsys_code/calendar_utils.py | 378 +++ solsys_code/campaign_filters.py | 28 + solsys_code/campaign_forms.py | 177 ++ solsys_code/campaign_gap.py | 278 +++ solsys_code/campaign_tables.py | 377 +++ solsys_code/campaign_urls.py | 34 + solsys_code/campaign_utils.py | 826 ++++++ solsys_code/campaign_views.py | 951 +++++++ .../commands/import_campaign_csv.py | 223 ++ .../commands/load_telescope_runs.py | 180 ++ .../sync_gemini_observation_calendar.py | 192 ++ .../commands/sync_lco_observation_calendar.py | 372 +++ .../0001_calendareventtelescopelabel.py | 23 + solsys_code/migrations/0002_campaignrun.py | 116 + ...mpaignrun_natural_key_unique_constraint.py | 20 + .../0004_campaignrun_window_schema.py | 144 ++ ...aign_run_window_start_end_null_together.py | 68 + ...al_obs_date_raw_and_window_needs_review.py | 27 + .../0007_campaignrun_contact_public_opt_in.py | 18 + solsys_code/mixins.py | 11 + solsys_code/models.py | 182 +- .../0002_observatory_timezone_seed.py | 17 + solsys_code/solsys_code_observatory/models.py | 50 +- .../tests/test_models.py | 32 +- .../tests/test_utils.py | 46 + solsys_code/solsys_code_observatory/utils.py | 64 +- solsys_code/solsys_code_observatory/views.py | 31 +- solsys_code/telescope_runs.py | 497 ++++ solsys_code/templatetags/__init__.py | 0 .../templatetags/calendar_display_extras.py | 198 ++ solsys_code/tests/test_admin.py | 76 + .../tests/test_calendar_display_extras.py | 189 ++ solsys_code/tests/test_calendar_template.py | 283 +++ solsys_code/tests/test_calendar_utils.py | 176 ++ solsys_code/tests/test_campaign_approval.py | 2208 +++++++++++++++++ solsys_code/tests/test_campaign_forms.py | 184 ++ solsys_code/tests/test_campaign_gap.py | 616 +++++ solsys_code/tests/test_campaign_models.py | 238 ++ .../tests/test_campaign_site_search.py | 291 +++ solsys_code/tests/test_campaign_submission.py | 280 +++ solsys_code/tests/test_campaign_views.py | 459 ++++ solsys_code/tests/test_import_campaign_csv.py | 925 +++++++ solsys_code/tests/test_load_telescope_runs.py | 341 +++ .../test_sync_gemini_observation_calendar.py | 234 ++ .../test_sync_lco_observation_calendar.py | 1133 +++++++++ solsys_code/tests/test_telescope_runs.py | 387 +++ solsys_code/tests/test_views.py | 20 +- .../tests/test_window_schema_migration.py | 121 + solsys_code/views.py | 117 +- src/fomo/settings.py | 13 + src/fomo/urls.py | 3 + src/templates/campaigns/approval_queue.html | 24 + src/templates/campaigns/campaign_list.html | 36 + .../campaigns/campaignrun_gap_analysis.html | 85 + .../campaigns/campaignrun_submit_form.html | 14 + .../campaigns/campaignrun_table.html | 54 + .../partials/site_search_results.html | 33 + .../campaigns/submission_thanks.html | 13 + .../solsys_code/partials/campaign_links.html | 1 + .../partials/campaigns_nav_link.html | 3 + .../observatory_create.html | 3 + .../tom_calendar/partials/calendar.html | 285 +++ src/templatetags/solsys_code_extras.py | 25 + 86 files changed, 21280 insertions(+), 37 deletions(-) create mode 100644 docs/design/GSD_Claude_notes.md create mode 100644 docs/design/eso_feasibility_spike.rst create mode 100644 docs/design/gsd_experiment.rst create mode 100644 docs/design/telescope_runs_calendar.rst create mode 100644 docs/design/tom_calendar_vs_yse_pz_calendar.rst create mode 100644 docs/design/uncertain_scheduling_spike.rst create mode 100644 docs/notebooks/ESO_How_to_download_data.ipynb create mode 100644 docs/notebooks/pre_executed/fixtures/campaign_sample.csv create mode 100644 docs/notebooks/pre_executed/import_campaign_csv_demo.ipynb create mode 100644 docs/notebooks/pre_executed/load_telescope_runs_demo.ipynb create mode 100644 docs/notebooks/pre_executed/sync_gemini_observation_calendar_demo.ipynb create mode 100644 docs/notebooks/pre_executed/sync_lco_observation_calendar_demo.ipynb create mode 100644 docs/notebooks/pre_executed/telescope_runs_demo.ipynb create mode 100644 solsys_code/calendar_urls.py create mode 100644 solsys_code/calendar_utils.py create mode 100644 solsys_code/campaign_filters.py create mode 100644 solsys_code/campaign_forms.py create mode 100644 solsys_code/campaign_gap.py create mode 100644 solsys_code/campaign_tables.py create mode 100644 solsys_code/campaign_urls.py create mode 100644 solsys_code/campaign_utils.py create mode 100644 solsys_code/campaign_views.py create mode 100644 solsys_code/management/commands/import_campaign_csv.py create mode 100644 solsys_code/management/commands/load_telescope_runs.py create mode 100644 solsys_code/management/commands/sync_gemini_observation_calendar.py create mode 100644 solsys_code/management/commands/sync_lco_observation_calendar.py create mode 100644 solsys_code/migrations/0001_calendareventtelescopelabel.py create mode 100644 solsys_code/migrations/0002_campaignrun.py create mode 100644 solsys_code/migrations/0003_campaignrun_natural_key_unique_constraint.py create mode 100644 solsys_code/migrations/0004_campaignrun_window_schema.py create mode 100644 solsys_code/migrations/0005_campaignrun_campaign_run_window_start_end_null_together.py create mode 100644 solsys_code/migrations/0006_campaignrun_original_obs_date_raw_and_window_needs_review.py create mode 100644 solsys_code/migrations/0007_campaignrun_contact_public_opt_in.py create mode 100644 solsys_code/mixins.py create mode 100644 solsys_code/solsys_code_observatory/migrations/0002_observatory_timezone_seed.py create mode 100644 solsys_code/telescope_runs.py create mode 100644 solsys_code/templatetags/__init__.py create mode 100644 solsys_code/templatetags/calendar_display_extras.py create mode 100644 solsys_code/tests/test_admin.py create mode 100644 solsys_code/tests/test_calendar_display_extras.py create mode 100644 solsys_code/tests/test_calendar_template.py create mode 100644 solsys_code/tests/test_calendar_utils.py create mode 100644 solsys_code/tests/test_campaign_approval.py create mode 100644 solsys_code/tests/test_campaign_forms.py create mode 100644 solsys_code/tests/test_campaign_gap.py create mode 100644 solsys_code/tests/test_campaign_models.py create mode 100644 solsys_code/tests/test_campaign_site_search.py create mode 100644 solsys_code/tests/test_campaign_submission.py create mode 100644 solsys_code/tests/test_campaign_views.py create mode 100644 solsys_code/tests/test_import_campaign_csv.py create mode 100644 solsys_code/tests/test_load_telescope_runs.py create mode 100644 solsys_code/tests/test_sync_gemini_observation_calendar.py create mode 100644 solsys_code/tests/test_sync_lco_observation_calendar.py create mode 100644 solsys_code/tests/test_telescope_runs.py create mode 100644 solsys_code/tests/test_window_schema_migration.py create mode 100644 src/templates/campaigns/approval_queue.html create mode 100644 src/templates/campaigns/campaign_list.html create mode 100644 src/templates/campaigns/campaignrun_gap_analysis.html create mode 100644 src/templates/campaigns/campaignrun_submit_form.html create mode 100644 src/templates/campaigns/campaignrun_table.html create mode 100644 src/templates/campaigns/partials/site_search_results.html create mode 100644 src/templates/campaigns/submission_thanks.html create mode 100644 src/templates/solsys_code/partials/campaign_links.html create mode 100644 src/templates/solsys_code/partials/campaigns_nav_link.html create mode 100644 src/templates/tom_calendar/partials/calendar.html diff --git a/.gitignore b/.gitignore index 39db0fb4..cdcd7a34 100644 --- a/.gitignore +++ b/.gitignore @@ -154,7 +154,15 @@ _html/ # Project initialization script .initialize_new_project.sh +docs/notebooks/data/** +docs/notebooks/eso_programmatic.py +Didymos_runs + # Serena MCP .serena/ -docs/notebooks/data/** +# Django MEDIA_ROOT — downloaded DataProduct files (src/fomo/settings.py) +src/data/ + +# Claude Code / GSD tooling (local install + machine-specific config) +.claude/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 13cd7dad..3c6dc6f7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: name: Clear output from Jupyter notebooks description: Clear output from Jupyter notebooks. files: \.ipynb$ - exclude: ^docs/pre_executed + exclude: ^docs/notebooks/pre_executed stages: [pre-commit] language: system entry: jupyter nbconvert --clear-output diff --git a/CLAUDE.md b/CLAUDE.md index afb988c3..ddcf0d91 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -89,7 +89,453 @@ Django/DB-dependent tests under the relevant app's `tests/` package. - Database is local SQLite (`src/fomo_db.sqlite3`); `DEBUG=True` and the secret key in `settings.py` are dev defaults — production overrides belong in a `local_settings.py` (imported at the end of `settings.py`). - Targets are `NON_SIDEREAL`; default target permission is `OPEN` and `AUTH_STRATEGY='READ_ONLY'`. +- **Target test factories:** when fixturing a `Target` in tests or notebooks, always use + `tom_targets.tests.factories.NonSiderealTargetFactory`, never `SiderealTargetFactory` — FOMO is + exclusively for Solar System / non-sidereal targets, so a sidereal fixture misrepresents what the + code actually handles. This applies to every GSD subagent (planner, executor, code-reviewer) that + writes or reviews test/demo code touching `Target`. - ruff config (`pyproject.toml`) follows Rubin DM style: many `N8xx` naming rules are intentionally ignored so astronomical variable names (e.g. `H`, `G`, `RA_deg`) are allowed. Format with single quotes. - pre-commit blocks direct commits to `main`, clears Jupyter notebook output, runs ruff, builds Sphinx docs, and runs the pytest suite. CI (`.github/workflows/`) tests Python 3.10–3.12. +- **Planning-doc terminology:** in CONTEXT.md/RESEARCH.md/PLAN.md/PATTERNS.md and other + `.planning/` artifacts, prefer plain English over DB jargon. Write "create or update" / + "find-or-create" / "create the record if missing, otherwise update it in place" instead of + "upsert". This applies to every GSD subagent (discuss-phase, researcher, planner, checker) — + they all read this file before producing planning docs. +- **Demo notebook companions are part of the deliverable**, not optional polish added after the + fact. Each of `solsys_code/telescope_runs.py`, + `solsys_code/management/commands/load_telescope_runs.py`, + `solsys_code/management/commands/sync_lco_observation_calendar.py`, and + `solsys_code/management/commands/sync_gemini_observation_calendar.py` has a paired demo notebook + under `docs/notebooks/pre_executed/` — `telescope_runs_demo.ipynb`, + `load_telescope_runs_demo.ipynb`, `sync_lco_observation_calendar_demo.ipynb`, and + `sync_gemini_observation_calendar_demo.ipynb` respectively — + that must stay in sync with the module's behavior. Any plan whose tasks change one of these + modules' behavior (new extraction logic, new parameters, new fixture shapes — not pure + refactors or typo fixes) must include its paired notebook in `files_modified` and add or update + cells exercising the new behavior with real executed output, regenerated via + `jupyter nbconvert --to notebook --execute --inplace` and committed (pre-commit clears notebook + output everywhere else, but `pre_executed/` copies are committed with output, per the + pre-commit convention noted above). When a new module gets its own demo notebook, extend this + list. This gap was hit twice already — Phase 5 (fixed after the fact via quick task + `260619-f7u`) and Phase 6 (fixed via quick task `260620-v9x`) — both times because the plan's + `files_modified` never scoped the notebook in. This applies to every GSD subagent touching + these modules: the planner (scope the paired notebook into `files_modified` and into a task up + front, not as a follow-up); the plan-checker (treat this as part of CLAUDE.md Compliance — + flag any plan that modifies one of the listed modules' behavior without its paired notebook in + `files_modified`); the executor (update the notebook as part of plan execution, not as an + afterthought); and the verifier (treat a missing or stale notebook update as a must-have gap, + not a nice-to-have, whenever the plan touched one of these modules). + + + +## Project + +**Telescope Runs Calendar — Stage 1 (Site/Ephemeris Helper)** + +A small helper module (`solsys_code/telescope_runs.py`) for FOMO that resolves a +telescope name to its observing site (via the existing `Observatory` model, +looked up by MPC obscode) and computes dip-corrected UTC sunset, sunrise, and +-15° dark-window crossing times for a given date. This is Stage 1 of the +"telescope runs on the calendar" feature (issue #37) — the foundation that +Stages 2-4 (classical run ingest, queue window banners, observation-record +sync) will build on. + +This GSD run is deliberately scoped to Stage 1 only: a self-contained, +well-specified unit used to trial the GSD discuss→plan→execute→verify loop on +this codebase before deciding whether to scale to the full 4-stage feature. + +**Core Value:** Stage 1 must do two things at once: produce sun-event times accurate to +within 2 minutes of the LCO skycalc reference tool (the feature actually +works), and be built end-to-end through GSD's discuss/plan/execute/verify +loop without the workflow stumbling on this repo's conventions (the +experiment actually validates). Either failing is a meaningful result. + +### Constraints + +- **Astronomy library**: `astropy` (`get_sun`, `AltAz`, `EarthLocation`) for + sun-position calculations — matches the design doc's validated approach. + +- **Timezones**: `zoneinfo` (stdlib, `tzdata` installed) for + `America/Santiago` and `Australia/Sydney`. + +- **Data source**: Site coordinates come from `Observatory` model records + (MPC obscode lookup), not hardcoded constants — Observatory records for the + 3 sites must exist (created via CreateObservatory form). + +- **Precision**: Sunset/sunrise must match LCO skycalc to <= 2 minutes; horizon + dip at 2402 m must be 1.44° ± 0.02°. + +- **Testing**: DB-dependent tests (Observatory lookups) go in + `solsys_code/tests/`, run with `./manage.py test solsys_code`. Quality gates: + `ruff check .` and `ruff format --check .` must stay clean. + + + + +## Technology Stack + +## Languages + +- Python 3.10+ - Core application and TOM Toolkit backend (Django-based) +- HTML/CSS/JavaScript - Django templates and frontend components (Bootstrap 4 based) + +## Runtime + +- Python 3.10, 3.11, 3.12 (tested across versions via GitHub Actions) +- `pip` - Python package management +- Lockfile: `pyproject.toml` (PEP 517/518 compliant) + +## Frameworks + +- Django 2.1+ (via TOM Toolkit) - Web framework for TOM Toolkit-based TOM application +- TOM Toolkit 2.31.4+ - Target and Observation Manager framework for Solar System object follow-up +- Django REST Framework - REST API support (`rest_framework`, `rest_framework.authtoken`) +- Django Crispy Forms (`crispy_forms`, `crispy_bootstrap4`) - Form rendering with Bootstrap 4 +- Bootstrap 4 (`bootstrap4`) - CSS framework +- Plotly (configured in settings, `PLOTLY_THEME = 'plotly_white'`) - Interactive visualization +- Django HTMX (`django_htmx`) - HTMX middleware for AJAX interactions +- Django ORM (via TOM Toolkit) - Database abstraction and models +- SQLite3 (default development) - File-based database backend +- pytest - Test runner +- pytest-cov - Code coverage reporting (`--cov` flags in GitHub workflows) +- setuptools 62+ - Package building +- setuptools_scm 6.2+ - Version management from git tags +- ruff 0.2.1+ - Linting and code formatting (via pre-commit) +- Sphinx - Documentation generation + +## Key Dependencies + +- tomtoolkit>=2.31.4 - TOM Toolkit framework for observatory management and observations +- tom_fink>=1.0.0 - Fink alert stream integration +- tom_alertstreams - Alert stream handling framework +- sorcha - Solar System object simulation and planning +- tom_eso - ESO (VLT) facility integration +- tom_observations - Core observation facilities (LCO, Gemini, SOAR) +- tom_catalogs - Catalog harvesters (JPL Horizons, MPC, SIMBAD, TNS) +- tom_registration - User registration and management +- django.contrib.auth - Authentication and authorization +- django.contrib.contenttypes - Content type framework +- django.contrib.sessions - Session management +- django.contrib.messages - Messaging framework +- django.contrib.sites - Multi-site framework +- django.contrib.admin - Django admin interface +- django.contrib.staticfiles - Static file serving +- django-extensions - Management commands and utilities +- django-guardian - Object-level permissions +- django-comments - Commenting system +- django-filters - Filtering for querysets +- django-tables2 - Table rendering +- django-gravatar - Gravatar integration +- django_gravatar - Avatar display +- numpy>1.24 - Numerical computing (for photometry/data processing) + +## Configuration + +- Environment variables via `os.getenv()` (see INTEGRATIONS.md for env var list) +- Django settings module: `src.fomo.settings` +- Local settings override via `local_settings.py` import (fallback: no error on missing) +- `pyproject.toml` - Main configuration (Python 3.10+ required, version dynamic via setuptools_scm) +- `.readthedocs.yml` - ReadTheDocs build configuration (Python 3.10, Sphinx) +- `.pre-commit-config.yaml` - Pre-commit hooks (ruff, pytest, Sphinx, validation) +- Ruff config in `pyproject.toml` - Format style (single quotes), line length 120 + +## Platform Requirements + +- Python 3.10, 3.11, or 3.12 +- Git (for setuptools_scm version management) +- SQLite3 support +- Pandoc (optional, for Jupyter notebook rendering in docs) +- Python 3.10+ +- SQLite3 or PostgreSQL (configurable via `DATABASES` setting) +- Static file serving setup (via Django `STATIC_URL`, `STATIC_ROOT`, `MEDIA_ROOT`) +- WSGI application server (configured at `src.fomo.wsgi.application`) +- Sphinx 2.1+ - HTML documentation generation +- ReadTheDocs - Hosted documentation platform + + + + + +## Conventions + +## Naming Patterns + +- Snake case for Python files (e.g., `test_ephem_utils.py`, `solsys_code_observatory`) +- Test files follow pattern: `test_*.py` (e.g., `test_models.py`, `test_views.py`, `test_utils.py`) +- Django app directories use descriptive snake_case with nested structures (e.g., `solsys_code/`, `solsys_code_observatory/`) +- Snake case throughout (e.g., `split_number_unit_regex`, `convert_target_to_layup`, `add_magnitude`, `add_sky_motion`) +- Private/internal functions use leading underscore (e.g., `_translate_constraints`) +- Method names follow Django conventions: `get_*`, `form_valid`, `setUp`, `handle` +- Snake case for all variables and parameters (e.g., `target_id`, `start_time`, `obscode`, `test_observatory`) +- Constants use UPPER_CASE (e.g., `AU_KM`, `SEC_PER_DAY`, `PI_OVER_2`, `MJD_TO_JD_CONVERSION`) +- Class attributes and properties follow snake case (e.g., `test_target`, `bary_vec`, `sun_dict`) +- Use modern Python type hints (Python 3.10+): `tuple[float, float]`, `dict[str, Any]`, `Optional[dict[str, Any]]` +- Return type annotations on methods: `def form_valid(self, form: EphemerisForm) -> HttpResponse:` +- Parameter type annotations where helpful: `def query(self, obscode: str, dbg: bool = False)` +- PascalCase for class names (e.g., `Observatory`, `EphemerisForm`, `JPLSBDBQuery`, `FakeSorchaArgs`) +- Inner/nested classes allowed (e.g., `Meta` in Django models) + +## Code Style + +- Line length: 120 characters (enforced by `ruff` and `black`) +- Quote style: Single quotes preferred by ruff formatter (e.g., `'ephem_form.html'`) +- Target Python version: 3.10+ +- Tool: `ruff` for linting and formatting +- Configuration in `pyproject.toml`: `[tool.ruff]` +- Pre-commit hook runs `ruff --fix` and `ruff-format` on all Python files +- Ruff lint rules include: E (pycodestyle), W (warnings), F (Pyflakes), N (pep8-naming), UP (pyupgrade), B (bugbear), SIM (simplify), I (isort) +- Per-file ignores for tests: `D101`, `D102` (missing docstrings) +- Per-file ignores for migrations: `D100`, `D101`, `D102`, `D103`, `E501`, `RUF012` +- Exceptions to naming rules: `N802`, `N803`, `N806`, `N812`, `N813`, `N815`, `N816`, `N999` (allow some variations for scientific/Numpy compatibility) + +## Import Organization + +- No path aliases defined in this project; relative imports use dot notation (e.g., `from .forms import`, `from .ephem_utils import`) +- Absolute imports from installed packages: `from tom_targets.models import Target` +- Profile: `black` +- Line length: 120 + +## Error Handling + +- Use generic `try/except` blocks for expected failures (e.g., `ValueError` when parsing time strings) +- Custom exceptions not extensively used; rely on built-in exceptions and Django exceptions +- Logging at `debug` level for expected failures: `logger.debug(f'Query failed with status {resp.status_code}')` +- Raise generic `Exception` for invariant violations (e.g., `raise Exception('Must provide target_id')`) + +## Logging + +- Get logger with `__name__`: `logger = logging.getLogger(__name__)` +- Log at `debug` level for diagnostic info: `logger.debug('No data found in results')` +- Test files can disable logging during test runs: `logging.disable(logging.CRITICAL)` +- Use f-strings for log messages: `logger.debug(f'Query failed with status {resp.status_code}')` + +## Comments + +- Comment non-obvious algorithmic steps (e.g., "Convert from heliocentric->barycentric using the Sun's position") +- Comment constants and their meaning (e.g., "Speed of light in km/s") +- Comment field meanings in data structures (e.g., chi-square values, degrees of freedom) +- Use comments to explain the "why" not the "what" (code should be readable, comments explain intent) +- Block comments above code sections that need context +- Not used (Python project, not TypeScript) +- Docstrings use Google-style format with `Args:`, `Returns:`, `Raises:` sections + +## Docstring Style + +- Google-style docstrings (not NumPy style, despite presence of NumPy code) +- Example from `ephem_utils.py`: +- Class docstrings: Simple one-liner (e.g., `"""View for making an ephemeris"""`) +- Method docstrings: Include Parameters and Returns sections +- One-liner functions may skip docstrings if name is self-explanatory + +## Function Design + +- Methods typically 10-50 lines +- Longer methods acceptable for view handlers (50-100+ lines) due to Django boilerplate +- Extract complex logic into helper functions +- Use keyword arguments for optional form parameters +- Type hints on parameters are encouraged +- Default parameters for optional behavior (e.g., `sun_dict=None`) +- Use type hints for return values +- Return `HttpResponse` from views +- Return `Optional[...]` for nullable types +- Tuples return multiple values with type hints: `-> tuple[float, float, float]` + +## Module Design + +- Modules export all public functions and classes +- No `__all__` definitions observed; relies on convention (no leading underscore = public) +- Internal/private use indicated by leading underscore +- No barrel files (index-style `__init__.py`) in use +- Package `__init__.py` files are typically empty or minimal + +## Code Quality Standards + +- `D101`: Missing docstring in public class (enforced except in tests) +- `D102`: Missing docstring in public method (enforced except in tests) +- `D103`: Missing docstring in public function +- Test files (`**/tests/*`) exempt from `D101`, `D102` requirements +- Avoid module-level mutable state +- Exception: `ephem_utils.py` loads and caches SPICE ephemeris kernels at module load time (acceptable for initialization) + + + + + +## Architecture + +## System Overview + +```text + +``` + +## Component Responsibilities + +| Component | Responsibility | File | +|-----------|----------------|------| +| Django App Setup | Entry point, URL routing, WSGI/ASGI | `src/fomo/settings.py`, `urls.py`, `wsgi.py`, `asgi.py` | +| Ephemeris Generation | Form handling and ephemeris request workflow | `solsys_code/views.py:MakeEphemerisView` | +| Ephemeris Display | CSV/HTML rendering of computed ephemeris | `solsys_code/views.py:Ephemeris` | +| Ephemeris Math | Orbital mechanics, coordinate transforms, magnitude calculation | `solsys_code/ephem_utils.py` | +| Observatory Management | CRUD for observatory sites (lat/lon, altitude) | `solsys_code/solsys_code_observatory/models.py`, `views.py` | +| JPL Discovery | Query JPL SBDB for solar system objects | `solsys_code/views.py:JPLSBDBQuery` | +| Form Validation & UI | Form fields and Crispy Forms layout | `solsys_code/forms.py`, `solsys_code_observatory/forms.py` | +| TOM Integration | App config hooks, template tags | `solsys_code/apps.py`, `src/templatetags/` | + +## Pattern Overview + +- Plugin architecture: FOMO extends TOM Toolkit as an installed app +- Django class-based views for form handling and data display +- Wrapper services (e.g., `FakeSorchaArgs`) abstract external library complexity +- Database-backed registry of observatories queried via Sorcha +- Template tag extensions for TOM integration points + +## Layers + +- Purpose: Render user-facing forms and results to HTML/CSV +- Location: `src/templates/` +- Contains: Form templates (`ephem_form.html`), result displays (`ephem.html`), observatory CRUD templates +- Depends on: Django template context from views, Crispy Forms layout +- Used by: Django view template rendering +- Purpose: Handle HTTP requests, validate forms, orchestrate business logic +- Location: `solsys_code/views.py`, `solsys_code/solsys_code_observatory/views.py` +- Contains: `MakeEphemerisView` (FormView), `Ephemeris` (View), `CreateObservatory` (CreateView), `ObservatoryList` (ListView), `ObservatoryDetailView` (DetailView) +- Depends on: Forms, models, ephem_utils, external APIs +- Used by: URL dispatcher +- Purpose: Define and validate input data, construct form layout +- Location: `solsys_code/forms.py`, `solsys_code/solsys_code_observatory/forms.py` +- Contains: `EphemerisForm` (date range, observatory selection, output options), `CreateObservatoryForm` (MPC code input) +- Depends on: Models, Crispy Forms helpers +- Used by: Views for initialization and validation +- Purpose: Compute ephemeris, transform coordinates, fetch external data +- Location: `solsys_code/ephem_utils.py`, `solsys_code/solsys_code_observatory/utils.py` +- Contains: Ephemeris computation functions, coordinate transforms (ERFA), magnitude calculation (add_magnitude, add_sky_motion), orbit conversion, n-body integration setup +- Depends on: Sorcha, ASSIST, SPICE, ERFA, Astropy +- Used by: Views, JPLSBDBQuery +- Purpose: Manage persistent storage and query interface +- Location: Django ORM models +- Contains: TOM Toolkit `Target` (external model), `Observatory` model with coordinate transforms +- Depends on: SQLite3, Django ORM +- Used by: Views, forms +- Purpose: Integrate with scientific libraries and remote APIs +- Location: Various dependencies (sorcha, rebound, assist, spiceypy, etc.) +- Contains: Orbital mechanics, coordinate geometry, SPICE kernel management, JPL/MPC API clients +- Depends on: External packages, network connectivity +- Used by: Business logic layer + +## Data Flow + +### Primary Request Path: Ephemeris Generation + +### Secondary Flow: Observatory Discovery & Management + +### Tertiary Flow: JPL Discovery + +- Request-local state: Form data, computed ephemeris held in request context +- Persistent state: Observatory models, Target models (TOM-managed) +- Module-level state: Sorcha ephemeris object (`ephem`), SPICE kernels (cached in `~/.cache/sorcha/`) + +## Key Abstractions + +- Purpose: Encapsulates observer position, time, and reference frames needed for coordinate transforms +- Examples: `EphemerisGeometryParameters` (from Sorcha), ERFA context setup in `ephem_utils.py` +- Pattern: Wrapper functions adapt external library interfaces to local use +- Purpose: Represents observing site with coordinate systems (geodetic, geocentric, parallax constants) +- Examples: `Observatory` model with methods `.to_parallax_constants`, `.to_geocentric()`, `.ObservatoryXYZ()` +- Pattern: Domain model with calculated properties and coordinate conversion methods +- Purpose: Intermediate representation of orbital elements in format Sorcha expects +- Examples: NumPy array constructed from Target fields +- Pattern: Adapter converting Django ORM objects to scientific library input +- Purpose: Encapsulates user input validation and context assembly +- Examples: `EphemerisForm` combines target ID, dates, step size, observatory selection +- Pattern: Crispy Forms layout with helper for custom HTML and actions + +## Entry Points + +- Location: `src/fomo/wsgi.py` +- Triggers: Web server (runserver, gunicorn, etc.) +- Responsibilities: Create Django WSGI application using `get_wsgi_application()` +- Location: `src/fomo/asgi.py` +- Triggers: ASGI server (Daphne, Hypercorn) for async support +- Responsibilities: Create Django ASGI application, configure for async +- Location: `manage.py` (project root) +- Triggers: `python manage.py ` +- Responsibilities: Execute management commands (migrate, runserver, etc.) +- Example: `python manage.py fetch_jplsbdb_objects` (custom command at `solsys_code/management/commands/fetch_jplsbdb_objects.py`) +- Location: `src/fomo/urls.py` +- Triggers: Django URL dispatcher +- Routes: `/ephem//` (Ephemeris view), `/targets//makeephem/` (MakeEphemerisView), `/observatory/` (solsys_code_observatory app), default TOM urls +- Location: `solsys_code/apps.py:SolsysCodeConfig` +- Method: `target_detail_buttons()` → injects "Make Ephemeris" button into TOM target detail view +- Method: `data_services()` → registers Fink data service for alert integration + +## Architectural Constraints + +- **Threading:** Django is single-threaded at the request level; ephemeris computation is synchronous. ASSIST and Sorcha operations run in-process and block request handling for large date ranges. +- **Global state:** Module-level Sorcha `ephem` object and SPICE kernels loaded once at startup (`solsys_code/ephem_utils.py:62-69`). This is memory-efficient but prevents kernel updates without restart. +- **Database:** SQLite3 has concurrent write limitations; production deployments should migrate to PostgreSQL. +- **Coordinate frames:** All ephemeris computations assume J2000 equatorial coordinates; celestial latitude/longitude are computed in ecliptic frame and transformed back. +- **Observatory selection:** Form restricts to observatories with `altitude > 0` (no submarine or underground sites). + +## Anti-Patterns + +### Inline Query URL Construction in JPLSBDBQuery + +```python + +``` + +### Form Initialization with Hardcoded Date Defaults + +```python + +``` + +### Silent Fallback in MPC Parallax Conversion + +```python + +``` + +## Error Handling + +- Form validation: `EphemerisForm.clean()` could validate date ranges (currently not implemented) +- View-level: `MakeEphemerisView.form_valid()` wraps ephemeris computation; unhandled exceptions bubble to Django error pages +- Model-level: `Observatory.from_parallax_constants()` silently returns None values (anti-pattern) +- External APIs: `JPLSBDBQuery.run_query()` handles HTTP errors but doesn't log them + +## Cross-Cutting Concerns + + + + + +## Project Skills + +No project skills found. Add skills to any of: `.claude/skills/`, `.agents/skills/`, `.cursor/skills/`, `.github/skills/`, or `.codex/skills/` with a `SKILL.md` index file. + + + + +## GSD Workflow Enforcement + +Before using Edit, Write, or other file-changing tools, start work through a GSD command so planning artifacts and execution context stay in sync. + +Use these entry points: + +- `/gsd:quick` for small fixes, doc updates, and ad-hoc tasks +- `/gsd:debug` for investigation and bug fixing +- `/gsd:execute-phase` for planned phase work + +Do not make direct repo edits outside a GSD workflow unless the user explicitly asks to bypass it. + + + + +## Developer Profile + +> Profile not yet configured. Run `/gsd:profile-user` to generate your developer profile. +> This section is managed by `generate-claude-profile` -- do not edit manually. + diff --git a/docs/conf.py b/docs/conf.py index 8cd71d6e..30dd1bb1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,6 +42,15 @@ templates_path = [] exclude_patterns = ['_build', '**.ipynb_checkpoints'] +# The pre-commit sphinx-build hook overrides exclude_patterns to skip +# notebooks/* for speed (avoids executing/rendering .ipynb files on every +# commit). That intentionally leaves docs/notebooks.rst's toctree entry +# pointing at an excluded document during local pre-commit builds, which +# would otherwise emit a 'toctree contains reference to excluded document' +# warning on every commit. Full builds (ReadTheDocs, CI) don't apply that +# override, so this only ever suppresses the pre-commit-local false positive. +suppress_warnings = ['toc.excluded'] + # This assumes that sphinx-build is called from the root directory master_doc = 'index' # Remove 'view source code' from top of page (for html, not python) @@ -56,3 +65,5 @@ autoapi_member_order = 'bysource' html_theme = 'sphinx_rtd_theme' +# Add following to allow notebook execution errors (e.g. interactive cells) +nbsphinx_allow_errors = True diff --git a/docs/design/GSD_Claude_notes.md b/docs/design/GSD_Claude_notes.md new file mode 100644 index 00000000..d0958f87 --- /dev/null +++ b/docs/design/GSD_Claude_notes.md @@ -0,0 +1,196 @@ +## Initial setup of Serena etc + +* Ensure `uv` is install from Astral +* Run `uv tool install -p 3.11 serena-agent` +* Run `serena init` +* Edit '~/.serena/serena_config.yml' and ensure large files e.g. FITS, HDF5 etc are excluded: +``` +ignored_paths: +- "*.fits" +- "*.fits.*" +- "*.bsp" +- "*.ecsv" +- "*.bin" +- "*.pt" +- "*.7z" +- "*.zip" +- "*.db" +- "*.hdf5" +``` +* Run `serena setup claude-code` +* To add to the global configuration to use for all probjects: +claude mcp add --scope user serena -- serena start-mcp-server --context claude-code --project-from-cwd +* Created an alias: +alias ccs='claude --system-prompt="$(serena prompts print-cc-system-prompt-override)"' +to launch Claude Code with Serena and get it, particulary Opus models, to use it per [Serena Claude Code docs][https://oraios.github.io/serena/02-usage/030_clients.html#claude-code] + + +# Notes on using GSD in Claude for issue37 branch + +## Phase 1 + +### Planning + +No notes. + +### Middle of Phase 1 (Start of execution) + ▶ Next Up + + Execute Phase 1 — run both plans (Wave 1 then Wave 2) + + /clear then: + + /gsd-execute-phase 1 + + ─────────────────────────────────────────────────────────────── + + Also available: + - cat .planning/phases/01-site-ephemeris-helper/*-PLAN.md — review plans + - /gsd-plan-phase 1 --research — re-research first + - /gsd-review --phase 1 --all — peer review plans with external AIs + + ─────────────────────────────────────────────────────────────── + + Note for execute-phase: the project's default node is v14, which is too old for gsd-tools.cjs (needs ES2021 ||=). Use + /home/tlister/.nvm/versions/node/v20.20.2/bin/node when invoking the tool, or prepend it to PATH. + + +### Security gate questions +❯ Great. I didn't understand the choice about the security gate; explain further + + Read 1 file, ran 1 shell command + +● Here's the full picture on what that security gate is and what running it would involve: + + What it is + + GSD has an optional "secure phase" workflow (/gsd:secure-phase 01) that runs a STRIDE-style threat-mitigation audit against a completed phase. + It's controlled by .planning/config.json: + + "security_enforcement": true, + "security_asvs_level": 1, + "security_block_on": "high" + + When enabled, it's meant to be part of the phase-completion checklist — execute-phase.md's aggregate_results step checks whether + {phase}-SECURITY.md exists, and if it doesn't, it's supposed to surface a suggestion to run it. + + How it would work for Phase 01 + + 1. It reads 01-01-PLAN.md/01-02-PLAN.md for a block (trust boundaries + a STRIDE threat register with dispositions). + 2. Our plans didn't author a block (telescope_runs.py isn't the kind of phase where STRIDE planning was prompted — it's a pure + computation module, no new attack surface, no user input, no auth/data boundaries). + 3. Because of that, register_authored_at_plan_time: false → the workflow falls into "retroactive-STRIDE mode": it spawns gsd-security-auditor, + which has to invent a threat register from scratch by reading the implementation files, then verify mitigations against that invented register. + 4. If it finds any "open" threats (even speculative/low-relevance ones for a module with no I/O, no auth, no untrusted input), it would either + present them for accept/block, or block phase advancement until resolved. + + Why I framed it as a "choice" + + For a module like telescope_runs.py — pure math over Observatory DB records and astropy, no network calls, no user input, no new permissions — + a STRIDE audit is largely going to produce either: + - nothing meaningful (good, but ~5 min subagent run for no signal), or + - low-value "threats" invented to fill the register (e.g. "what if Observatory.timezone is attacker-controlled" — which isn't really a security + boundary here, just the WR-03 robustness issue we already fixed). + + So the choice was really: run the formal gate now (costs a subagent run, may produce noise, but satisfies the config's enforcement requirement + and creates 01-SECURITY.md) vs. skip it (the config says it's required, but the phase's actual risk profile doesn't warrant it). + + I didn't make that call for you — I just surfaced that the gate exists and is unsatisfied. If you want, I can run /gsd:secure-phase 01 now; + given what we've already covered in code review (WR-02/03 input-validation gaps, now fixed), I'd expect it to come back clean or with only + "accepted risk" items. + +### Usage (at end of day/session) + + Settings Status Config Usage Stats + + Session + + Total cost: $4.51 + Total duration (API): 9m 41s + Total duration (wall): 27m 52s + Total code changes: 146 lines added, 32 lines removed + Usage by model: + claude-sonnet-4-6: 2.7k input, 47.4k output, 6.6m cache read, 399.0k cache write ($4.51) + + Current session + ██████████████████████████████████████████████████ 100% used + Resets 5:39pm (America/Los_Angeles) + + Current week (all models) + █████████████▌ 27% used + Resets Jun 16, 9:59pm (America/Los_Angeles) + + What's contributing to your limits usage? + Approximate, based on local sessions on this machine — does not include other devices or claude.ai + + Last 24h · these are independent characteristics of your usage, not a breakdown + + 90% of your usage came from subagent-heavy sessions + Each subagent runs its own requests. Be deliberate about spawning them — and + consider configuring a cheaper model for simpler subagents. + + 11% of your usage was at >150k context + Longer sessions are more expensive even when cached. /compact mid-task, /clear + when switching to new tasks. + + 49% of your usage came from plugin "gsd" + Review what this plugin contributes — its agents, skills, and MCP tools all + count toward your limit. + + Skills % of usage + /gsd:plan-phase 9% + /gsd:code-review 5% + /gsd:discuss-phase 4% + /gsd:new-project 3% + /gsd:execute-phase 3% + /gsd:map-codebase 1% + /gsd:phase 1% + + Subagents % of usage + gsd:plan-phase 9% + gsd:execute-phase 8% + gsd-executor 6% + gsd:code-review 3% + gsd:map-codebase 2% + gsd-roadmapper 1% + + Plugins % of usage + gsd 49% + + d to day · w to week + + Usage credits + █████▋ 11% used + $4.50 / $40.00 spent · Resets Jul 1 (America/Los_Angeles) + + +## Fixing up issues after phase 3 +Run: +/gsd-code-review 03 --fix + +After that, plus an additional fix to truncate microseconds and change the example Magellan instrument from LDSS3 +● Both phases of v1.1 are complete and all code review fixes have been applied. The STATE.md is slightly stale (it still +says "verifying"), but in practice the branch is ready. Your options: + +1. Create a PR now — everything is in good shape; the verification was already done before the code review, and the +fixes were all polish (error handling, temp-dir leak, encoding, microseconds). Run /gsd-pr-branch or just gh pr +create. +2. Re-verify first (/gsd-verify-work) — if you want a formal second pass on the verification artifact now that CR-01 +and the microseconds fix have changed behavior. Low stakes since none of the fixes changed correctness of the +astronomy or the idempotency logic. +3. Close the milestone (/gsd-complete-milestone) — archives the v1.1 planning artifacts and sets up for the next +milestone (Stage 3 queue-window banners, or Stage 4 observation-record sync, depending on what you want to tackle +next). + + +## Phase 4 + +▶ Next Up — Telescope Runs Calendar + +Phase 4: LCO Queue Sync Command — sync_lco_observation_calendar --proposal syncs LCO ObservationRecords to CalendarEvents with banner → placed-block → terminal-state lifecycle + +/clear then: + +/gsd-discuss-phase 4 — gather context and clarify approach before planning + +Also: /gsd-plan-phase 4 — skip discussion, plan directly diff --git a/docs/design/design.rst b/docs/design/design.rst index 06943994..ebb3d317 100644 --- a/docs/design/design.rst +++ b/docs/design/design.rst @@ -32,3 +32,15 @@ Component .. image:: /design/SSSC-Followup_Observations_of_Moving_Objects-Component.png :alt: Component diagram for FOMO + +Design Notes +------------ + +.. toctree:: + :maxdepth: 1 + + telescope_runs_calendar + tom_calendar_vs_yse_pz_calendar + gsd_experiment + eso_feasibility_spike + uncertain_scheduling_spike diff --git a/docs/design/eso_feasibility_spike.rst b/docs/design/eso_feasibility_spike.rst new file mode 100644 index 00000000..01c63193 --- /dev/null +++ b/docs/design/eso_feasibility_spike.rst @@ -0,0 +1,174 @@ +ESO/VLT Calendar Sync — Feasibility Spike +========================================== + +This document records the feasibility spike investigating whether ESO/VLT +observation sync can work at all for FOMO's telescope-runs calendar. It was +written after an investigation (2026-07-01) that connected to the real ESO +Phase 2 (P2) API with production Paranal credentials, captured live OB +status/execution data, and confirmed a headless credential-sourcing path for +a future management command. No sync command was built during this spike — +the deliverable is this decision record and its full-detail companion, +``13-DECISION.md`` (originally at +``.planning/phases/13-eso-feasibility-spike/13-DECISION.md``; this project's +milestone-archival workflow moves completed phase directories to +``.planning/phases-archive/`` once their milestone closes, so check there +first if the original path no longer resolves). + +Background +---------- + +FOMO already syncs queue-scheduled observation blocks to the calendar for +LCO (queue banner -> placed block, Stage 3) and Gemini (submission-time +window banner, Stage 3b). ESO/VLT observing is different again: Paranal +(VLT) and La Silla (NTT) both run in **Service Mode**, where Paranal Science +Operations staff choose which Observation Blocks (OBs) to execute in real +time based on current conditions — there is no advance per-OB schedule +published ahead of time, unlike LCO's queue scheduler. (ESO/NTT *classical* +scheduling — whole nights assigned in advance — is a separate, already-solved +problem handled by Stage 2's ``load_telescope_runs``; this spike is about +*queue*/OB-level sync, not classical nights.) + +The core constraint that reshaped this milestone: the installed +``tom_eso==0.2.4`` plugin cannot create ``ObservationRecord`` rows through +the standard TOM submission flow (``submit_observation()`` always returns an +empty ID list) and does not implement the TOM Toolkit status/URL interface +(``get_observation_status()``, ``get_observation_url()``, and +``data_products()`` all raise ``NotImplementedError``). This meant the usual +"read ``ObservationRecord`` rows, sync them to the calendar" pattern used by +LCO and Gemini had no data to read for ESO — the open question this spike +had to answer was whether *any* real ESO data was reachable at all, and if +so, how. + +Key finding +----------- + +**Bypass:** sync straight from the ESO P2 API (``p2api``) to +``CalendarEvent``, skipping ``ObservationRecord`` for ESO entirely. +Real Paranal production credentials connect and return real OB data through +direct P2 API calls (``getOB()``, ``getOBExecutions()``, +``getNightExecutions()``); a headless credential-sourcing path (a +``FACILITIES['ESO']`` settings entry, mirroring LCO/SOAR/GEM) was confirmed +viable without needing the session-bound, per-user ``ESOProfile`` path. No +evidence was gathered (or needed to be gathered, given this phase's +read-only guardrail) for creating hand-built ``ObservationRecord`` rows — +that is the Bridge option, and this spike's real-data path never touched it. + +Investigation summary +---------------------- + +.. list-table:: + :header-rows: 1 + :widths: 22 20 58 + + * - Capability + - Status + - Notes + * - Paranal (VLT) P2 connection + - Working + - ``ESOAPI(environment='production', ...)`` connects and returns real + data; confirmed via live ``getOB()``/``getNightExecutions()`` calls. + * - La Silla (NTT) P2 connection via ``tom_eso.eso_api.ESOAPI`` + - Fails (wrapper bug, not API access) + - Fails at ``ESOAPI.__init__`` because it unconditionally constructs a + Phase-1 (``p1api``) connection first, and ``p1api``'s ``API_URL`` has + no ``production_lasilla`` entry. ``p2api``'s own ``API_URL`` *does* + support ``production_lasilla``. Filed upstream as + `TOMToolkit/tom_eso#55 `_. + * - La Silla (NTT) P2 connection via direct ``p2api`` bypass + - Connects; La Silla-specific data unconfirmed + - ``p2api.ApiConnection('production_lasilla', ...)`` (bypassing + ``ESOAPI``/``p1api``) connects without error and returns real data — + confirming the wrapper-bug diagnosis. The one run returned was a + Paranal-instrument run already seen under ``production``, so this + proves the connection path is open but does not yet confirm distinct + La-Silla-sourced OB data is reachable for this account. + * - ``get_observation_status()`` / ``get_observation_url()`` / + ``data_products()`` (``tom_eso``-level) + - Not usable; unimplemented in ``tom_eso`` + - All three raise ``NotImplementedError`` in the installed + ``tom_eso==0.2.4``. Not needed under the Bypass path — a future sync + command reads ``obStatus`` directly from ``p2api``'s ``getOB()`` / + ``getOBExecutions()`` / ``getNightExecutions()`` responses instead. + * - Real OB status data (``p2api``-level, direct) + - Reachable + - A never-executed OB returned ``obStatus='P'`` with an empty executions + list; a separately-queried, already-executed OB's per-night execution + record returned ``obStatus='M'`` (Must Repeat) with a concrete + ``from``/``to`` time window and ``grade='X'``. + * - Headless credential-sourcing (for a future management command) + - Viable + - Direct ``ESOAPI(environment, username, password)`` construction from + environment-variable-supplied credentials works with no active + Django session and no ``ESOProfile`` involved — the same pattern + LCO/SOAR/GEM already use via ``FACILITIES[...]`` settings entries. + +ESO P2 ``obStatus`` vocabulary (12 codes) +-------------------------------------------- + +If a future sync command layers OB status onto the banner (status-aware +sync, see Future scope below), this is the vocabulary it would map, entirely +distinct from LCO's or Gemini's terminal-state sets: + +.. list-table:: + :header-rows: 1 + :widths: 12 48 20 + + * - Code + - Meaning + - Terminal? + * - ``P`` + - Partially defined (just created) + - No + * - ``D`` + - Defined (passed certification, ready for review) + - No + * - ``-`` + - Rejected (needs user attention) + - No + * - ``R`` + - Review (under revision by support astronomer) + - No + * - ``+`` + - Accepted (ready to be observed) + - No + * - ``C`` + - Completed (executed successfully, will not repeat) + - Yes + * - ``X`` + - Executed (successfully completed, can repeat — e.g. visitor mode) + - Yes (per-execution) + * - ``M`` + - Must repeat (executed outside constraints, will be requeued) + - No (requeues) + * - ``A`` + - Aborted during execution (will be requeued) + - No (requeues) + * - ``F`` + - Failed (absolute time window expired; read-only, irreversible) + - Yes + * - ``K`` + - Kancelled (support-astronomer set, irreversible) + - Yes + * - ``T`` + - Terminated (run terminated, irreversible) + - Yes + +Future scope +------------ + +See ``13-DECISION.md`` (path note above) for the full recommendation +rationale and future-sync sketch. In brief, a future +``sync_eso_observation_calendar``-style command (not built in this +milestone) would: + +* Reuse ``solsys_code/calendar_utils.py:insert_or_create_calendar_event()`` + unchanged — it is already facility-agnostic. +* Key idempotency on a synthetic identifier, ``ESO:{p2_environment}/{obId}``, + following the precedent set by Gemini's ``GEM:{prog}/{observation_id}``. +* Choose between a banner-only window sync (OB run-period dates, no status) + or a status-aware sync (layering the ``obStatus`` vocabulary above onto the + banner) — the latter is real-data-supported by this spike but requires a + polling-window policy (which night(s) to check per OB) that this + investigation did not need to resolve. + +This is input to a future milestone's requirements, not implemented here. diff --git a/docs/design/gsd_experiment.rst b/docs/design/gsd_experiment.rst new file mode 100644 index 00000000..80bc73bf --- /dev/null +++ b/docs/design/gsd_experiment.rst @@ -0,0 +1,137 @@ +GSD Experiment: Spec-Driven Feature Development +=============================================== + +This document records an assessment (2026-06-11) of using GSD as an experiment +to design and build a new FOMO feature from scratch, and the FOMO-specific +considerations to plan around before trying it. + +What GSD is +----------- + +There are two closely related GSD projects: + +* `get-shit-done `_ by TÂCHES — the + original (~54K stars as of June 2026). +* `gsd-core `_ from + `Open GSD `_ — a multi-runtime open + fork whose motto is "Git. Ship. Done." + +Both implement the same idea: a disciplined **discuss → plan → execute → +verify → ship** loop per phase, where heavy research, planning, and execution +run in fresh-context subagents (avoiding "context rot" as an AI session's +context window fills), and structured artifacts (``STATE.md``, ``CONTEXT.md``, +per-phase task plans) carry memory across sessions. Each task gets an atomic +git commit. + +Why FOMO is a good testbed +-------------------------- + +* **A ready-made spec exists.** The telescope-runs calendar design doc is + committed (issue #37, nothing built yet, Stage 1 = ephemeris helper). GSD's + discuss/plan phase normally has to extract a spec from conversation; feeding + it a finished design doc tests how faithfully the system decomposes and + executes a spec, rather than testing one's ability to articulate + requirements on the fly. + +* **The feature is multi-stage by design.** Staged work (ephemeris helper → + calendar model → views) maps directly onto GSD's phase loop, which is where + the system earns its keep. A one-file feature would not be a meaningful + test. + +* **FOMO's work pattern matches GSD's pitch.** Development happens across + many sessions over weeks; GSD's persistent artifacts are built exactly for + surviving session boundaries, the thing plain AI-assistant sessions are + worst at. + +* **Solo-project benefit:** the mandatory verify step is a stand-in for the + code reviewer the project does not have. + +FOMO-specific friction to plan around +------------------------------------- + +* **The ~1.6 GB SPICE kernel import side effect.** Any subagent that runs + ``./manage.py test`` (or imports ``solsys_code.ephem_utils``) triggers + ``fomo_furnish_spiceypy()``. The download is cached in ``~/.cache/sorcha/`` + so it is a one-time cost per machine, but the first verify phase will be + slow, and the ASSIST ephemeris build cost recurs. Warm the cache before + starting. + +* **The two-test-suite split.** Fresh-context subagents will reflexively run + ``python -m pytest``, which does not collect the ``solsys_code/`` Django app + tests. ``CLAUDE.md`` documents this and GSD subagents do read project + instructions, but each task plan should explicitly name which suite to run — + this is the most likely silent failure mode of the experiment. + +* **Pre-commit cost × atomic commits.** The pre-commit hooks run ruff, a + Sphinx build, and the pytest suite on every commit. GSD commits per-task, + so a 12-task phase pays that hook chain 12 times. Tolerable, but expect it. + +* **Start clean.** Run the experiment on a fresh branch off ``main`` so GSD's + commit stream does not tangle with in-flight work (e.g. the tomtoolkit 3.0 + migration). Decide up front whether GSD's planning artifacts (its + ``.gsd/``/planning directory) get committed or gitignored. + +Model choice on a Claude Pro plan +--------------------------------- + +(Assessed 2026-06-12.) GSD assigns models per phase via **model profiles** — +six slots (planning / discuss / research / execution / verification / +completion) accepting tier aliases (``opus``, ``sonnet``, ``haiku``, +``inherit``). The built-in profiles are ``quality`` (all Opus), ``balanced`` +(Opus for planning only, Sonnet elsewhere — GSD's default), ``budget`` (Sonnet +for code, Haiku for research/verification), and ``adaptive``. + +What a Pro plan provides (per the official +`Claude Code model docs `_): +the tier default is **Sonnet 4.6** (Max/API accounts default to Opus 4.8). +Opus access on Pro is limited at best and drains the 5-hour usage window +several times faster than Sonnet; Opus with 1M context requires extra usage +credits. **Fable 5 is effectively out of reach on Pro** — it is not the +default on any plan, is priced above Opus tier, and the ``best`` alias falls +back to the latest Opus where Fable access is absent. + +Recommendation for the GSD experiment on Pro: + +* **Sonnet 4.6 as the workhorse.** GSD's fresh-context subagents with small, + atomic task plans are exactly the regime where Sonnet performs closest to + Opus; the binding constraint on Pro is token *volume* (every phase spawns + multiple subagents), not per-task intelligence. + +* **Profile = ``balanced`` if Opus works on the account** — spend Opus only on + the planning agent, where the issue #37 design doc gets decomposed into task + plans (the one step where extra reasoning compounds). Claude Code's + ``opusplan`` alias is the same idea at the harness level. + +* **Fall back to ``budget`` if Opus is unavailable or limits bite.** The + verify phase is mostly "run the right test suite and read the output", which + does not need Opus. + +* **Avoid ``quality`` (all-Opus) on Pro** — it would exhaust a usage window + mid-phase, and a GSD run interrupted by rate limits is worse than one run on + Sonnet throughout. + +* Leave Sonnet 4.6's effort parameter at its default ``high``; ``max`` is + session-only and token-hungry. + +Recommendation +-------------- + +Do it, but scope the first run to **Stage 1 (the ephemeris helper) only**. +That is a self-contained unit with a clear spec, it touches the gnarliest part +of the codebase (``ephem_utils``), and it will surface all the friction points +above cheaply. If GSD handles the test-suite split and the heavy-import quirk +gracefully on Stage 1, scale it to the full calendar feature; if it stumbles, +the lesson — where its fresh-context model breaks on a repo with non-obvious +conventions — costs only one small phase. + +References +---------- + +* `gsd-build/get-shit-done `_ +* `GSD User Guide `_ +* `open-gsd/gsd-core `_ +* `Open GSD — Git. Ship. Done. `_ +* `Augment Code on GSD `_ +* `Claude Code model configuration `_ +* `GSD model profiles and cost optimization `_ +* `Configuring GSD model profiles `_ diff --git a/docs/design/telescope_runs_calendar.rst b/docs/design/telescope_runs_calendar.rst new file mode 100644 index 00000000..b69ae588 --- /dev/null +++ b/docs/design/telescope_runs_calendar.rst @@ -0,0 +1,347 @@ +Telescope Runs on the Calendar +============================== + +This document records the feasibility study and implementation plan for showing +follow-up telescope runs on the TOM Toolkit calendar (``tom_calendar``). It was +written after a research spike (2026-06-10) that validated the astronomy and the +data model end-to-end. + +Background +---------- + +FOMO coordinates follow-up of Solar System targets across several telescopes. +The scheduled time on those telescopes falls into two scheduling models: + +* **Classically-scheduled nights** — whole nights assigned to a programme in + advance. FOMO currently cares about two such telescopes in Chile: + + * **NTT / EFOSC2** at ESO La Silla Observatory. + * **Magellan** (Baade / Clay) at Las Campanas Observatory. + +* **Queue-scheduled blocks** — short blocks placed dynamically by a scheduler + within an eligible window: + + * **FTS / MuSCAT4** at Siding Spring Observatory, operated by Las Cumbres + Observatory (LCOGT). Blocks are roughly six hours per night, schedulable + across a range of nights bounded by Moon phase. + +The goal is to surface this allocated time on the new ``tom_calendar`` calendar +(live during development at ``/calendar/``) so that follow-up can be planned +against known telescope access. + +Key finding +----------- + +**The feature is feasible with no changes to** ``tom_calendar`` **and no +database migrations.** The stock :class:`tom_calendar.models.CalendarEvent` +already carries ``title``, ``description``, ``start_time``, ``end_time``, +``url``, ``telescope``, ``instrument``, ``proposal``, ``user`` and a +``target_list`` foreign key. A telescope run maps directly onto it. +``tom_calendar`` is a third-party install (part of the TOM Toolkit), **not** +FOMO/``tom_jpl`` code, so the design deliberately reuses existing fields rather +than patching the package. + +The Data Model +-------------- + +``CalendarEvent`` fields and how they surface in the UI (verified by reading the +``tom_calendar`` templates): + +.. list-table:: + :header-rows: 1 + :widths: 18 22 30 30 + + * - Field + - Type + - Where it shows + - Use for runs + * - ``title`` + - ``CharField(200)`` + - **Grid label** (truncated ~16 chars) and edit modal + - Short label; the only place to surface status at a glance + * - ``description`` + - ``TextField`` + - Edit modal only + - Dark window (UTC), original run string, status, notes + * - ``start_time`` / ``end_time`` + - ``DateTimeField`` + - Grid (timed events show start) and modal + - Sunset / sunrise (classical) or block bounds (queue) + * - ``telescope`` / ``instrument`` + - ``CharField(200)`` + - Edit modal + - ``NTT`` / ``EFOSC2`` etc., kept clean and queryable + * - ``url`` + - ``URLField`` + - Edit modal (link) + - Click-through; **idempotency key** for synced blocks + * - ``proposal`` / ``user`` + - ``CharField(200)`` + - Edit modal + - Programme ID / observer, if available + * - ``target_list`` + - FK ``TargetList`` + - **Grid badge** and modal + - Attach the night's targets + +The related ``EventTodo`` model (per-event checklist; its active count renders on +the grid) and a read-only ``color`` property (derived from ``pk``, so it cannot +be set to encode status) are available but not central to this design. + +Because only ``title`` and the ``target_list`` badge render on the calendar +grid, any status that must be glanceable belongs in ``title``; everything else +lives in ``description`` and the typed fields, visible on click-through. + +Astronomy: Night Boundaries +--------------------------- + +For classical runs each night becomes one event spanning **sunset to sunrise**, +with the **-15 deg dark window** recorded in the description. (-15 deg is a +deliberate FOMO choice for faint targets; textbook nautical twilight is -12 deg +and astronomical is -18 deg.) + +Sun altitudes are computed with ``astropy`` (``get_sun`` -> ``AltAz`` at the site +``EarthLocation``). Two corrections matter: + +* **Refraction + solar semidiameter:** geometric sunrise/sunset uses a threshold + Sun altitude of -0.833 deg. +* **Horizon dip** from the observatory's elevation. At ~2400 m the visible + horizon is depressed, so sunset is later and sunrise earlier. The dip is the + Nautical Almanac formula + + .. math:: \mathrm{dip} = 1.76' \sqrt{h_\mathrm{metres}} + + derived from spherical geometry (:math:`\theta \approx \sqrt{2h/R}`, + :math:`R = 6371` km) with terrestrial refraction :math:`k \approx 1/6` folded + in. At 2402 m this is 1.44 deg, so the sunset/sunrise threshold is + -(0.833 + 1.44) = -2.27 deg. The dip is **not** applied to the -15 deg window + (the Sun is nowhere near the visible horizon there). + +**Validation.** Computed times were checked against Las Campanas Observatory's +own ephemeris tool (John Thorstensen's *skycalc*, served from +``https://www.lco.cl/eph/``) for June 2026: + +* Sunset / sunrise agree to **<= 1 minute** once the horizon dip is applied + (without it there was a consistent ~8 minute error). +* Astronomical twilight (-18 deg) agrees to **<= 1 minute**, confirming the + twilight solver and hence the -15 deg numbers. +* The tool's "Chilean time (4 hr W)" for June matches ``zoneinfo`` returning + UTC-4 for ``America/Santiago``. + +Observatory Sites +----------------- + +.. list-table:: + :header-rows: 1 + :widths: 22 12 12 12 22 20 + + * - Telescope @ Site + - Lat (deg) + - Lon (deg) + - Alt (m) + - Timezone + - Source + * - Magellan @ Las Campanas + - -29.0146 + - -70.6926 + - 2402 + - ``America/Santiago`` + - Validated vs Las Campanas skycalc tool + * - NTT @ La Silla + - -29.2567 + - -70.7300 + - 2347 + - ``America/Santiago`` + - astropy ``of_site('lasilla')`` + * - FTS @ Siding Spring + - -31.2734 + - 149.0612 + - 1149 + - ``Australia/Sydney`` + - astropy ``of_site('Siding Spring Observatory')`` + +Chile uses ``America/Santiago`` (DST: UTC-4 in austral winter, UTC-3 summer). +NSW uses ``Australia/Sydney`` (DST: AEST UTC+10 winter, AEDT UTC+11 summer). +Both are handled by ``zoneinfo``; ``tzdata`` is installed. At +149 deg longitude +a Siding Spring night sits *within* a single UTC date (~07:00-21:00 UTC) rather +than straddling UTC midnight as Chilean nights do. + +Classical Run Input Format +-------------------------- + +Runs are recorded as free text lines, ``telescope instrument [status] daterange +[(status)]``. Observed examples:: + + NTT EFOSC2 allocation 9-13 July + Magellan IMACS 13-19 July (proposed) + Magellan Proto-Lightspeed Jul 8-12 (proposed) + +Parsing rules (prototype verified against all three lines): + +* ``telescope`` = token 0 (maps to a site: ``NTT`` -> La Silla, ``Magellan`` -> + Las Campanas); ``instrument`` = token 1 (may be hyphenated, e.g. + ``Proto-Lightspeed``). +* Date range ``D-D`` with the month name appearing **before or after** the day + range (``9-13 July`` and ``Jul 8-12`` both occur). +* **No year** is given. Default to the current year, with special handling for + runs beginning in late December (roll into the next year). +* Status comes from a parenthetical ``(...)`` or a bare known word + (``allocation``, ``proposed``, ...). + +**Night convention (confirmed from the Las Campanas telescope schedule).** Run ``Start`` +and ``End`` dates are **both observing nights** by evening date; the next run +begins the day *after* ``End``. (E.g. a Las Campanas run ``Start 2026-06-08 / +End 2026-06-10`` covers the nights of the evenings of the 8th, 9th and 10th; a +different instrument and PI appear on Baade from the 11th.) Therefore a run from +evening ``S`` to evening ``E`` yields ``E - S + 1`` nights, one event per evening +date ``d`` with ``start = sunset(d)``, ``end = sunrise(d+1)``. Consecutive runs +tile without overlap. + +**ESO / La Silla night convention (Tatoo noon-to-noon).** This both-inclusive +rule is specific to Las Campanas. ESO's *Tatoo* scheduling tool instead +displays a run's date range with an **End date that is the noon-to-noon closing +boundary of the last night, not itself an observing night** (e.g. +``NTT ... 9-13 July`` is *4.0 nights* in Tatoo: the evenings of the 9th, 10th, +11th and 12th; the 13th is only the closing noon of the night of the 12th). So +for ESO sites (listed in ``telescope_runs.ESO_NOON_TO_NOON_SITES``) a run from +evening ``S`` to boundary ``E`` yields ``E - S`` nights, one fewer than the Las +Campanas convention above. ``load_telescope_runs._iter_run_nights`` applies +whichever convention matches the run's site. + +Queue Runs: #1 + #3 +------------------- + +Queue scheduling is softer than a classical night, so it is represented two ways: + +**#1 - Window banner (the plan).** One multi-day ``CalendarEvent`` per queue +run. ``start_time`` / ``end_time`` are the eligible window, supplied **in UTC** +(no timezone conversion). ``telescope='FTS'``, ``instrument='MuSCAT4'``, +``title='FTS/MuSCAT4 (queue)'``; description records ~6 h/night, total hours and +the Moon constraint. No per-night blocks are fabricated, since the scheduler may +use only some nights. + +**#3 - Real blocks (the truth), synced from observation records.** FOMO already +has ``LCOFacility`` configured (``settings.py``), so submitted observations exist +as :class:`tom_observations.models.ObservationRecord` rows with +``facility='LCO'``, ``target``, ``parameters``, ``observation_id``, ``status``, +``scheduled_start`` and ``scheduled_end``. The LCO/OCS facility's +``update_observation_status`` populates the scheduled times from the current +block and reports status from the vocabulary ``PENDING``, ``COMPLETED`` and the +terminal states ``WINDOW_EXPIRED`` / ``CANCELED`` / ``FAILURE_LIMIT_REACHED`` / +``NOT_ATTEMPTED``. + +For each LCO record with scheduled times present, the sync **creates or updates** a +``CalendarEvent`` at the real block time. The idempotency key is the ``url`` +field set to the canonical portal URL +``https://observe.lco.global/requestgroups//`` — unique per +request and doubling as the click-through. Status is reflected in ``title`` (and +in full, with ``observation_id``, in ``description``); terminal-failure records +either delete their block or render it struck-through. + +Implementation Plan +------------------- + +The work is staged so each step is independently testable. + +**Stage 1 — site / ephemeris helper.** A small module (e.g. +``solsys_code/telescope_runs.py``) with: + +* a ``SITES`` registry mapping telescope name -> (site label, ``EarthLocation``, + timezone); +* ``sun_event(site, date, kind)`` returning UTC sunset/sunrise/-15 deg-dark + crossings using the dip-corrected thresholds above. + +**Stage 2 — classical ingest command.** ``load_telescope_runs`` management +command (modelled on ``fetch_jplsbdb_objects``): parse run lines, expand each to +one ``CalendarEvent`` per night via Stage 1, idempotent on re-run. + +**Stage 3 — queue window banners (#1).** Extend the command (or add a flag / +input mode) to accept FTS UTC windows and create one banner ``CalendarEvent`` per +queue run. + +**Stage 4 — observation-record sync (#3).** ``sync_lco_observation_calendar`` +(management command, or a ``post_save`` signal on ``ObservationRecord`` in +``solsys_code`` since FOMO owns that code), run after +``update_all_observation_statuses``; create or update ``CalendarEvent`` rows keyed on the +portal ``url``. + +Success Criteria +---------------- + +Testable, verifiable acceptance criteria. Django-DB tests live under +``solsys_code/tests/`` (run with ``./manage.py test``); pure helpers may also be +unit-tested. + +*Stage 1 — ephemeris helper* + +#. For Las Campanas, June 2026, computed sunset and sunrise (dip-corrected) are + within **2 minutes** of the Las Campanas *skycalc* tool for at least the sample nights + Jun 1/10/20/30. (Observed: <= 1 min.) +#. Computed astronomical twilight (-18 deg) for Jun 10 2026 is within 2 minutes + of the tool's ``twi.end`` / ``twi.beg`` (19:16 / 06:08 local). +#. ``America/Santiago`` resolves to UTC-4 in June and UTC-3 in January; + ``Australia/Sydney`` to UTC+10 in July and UTC+11 in January (asserted). +#. The horizon-dip helper returns 1.44 deg +/- 0.02 at 2402 m. + +*Stage 2 — classical ingest* + +#. Parsing the three sample lines yields the expected + ``(telescope, instrument, status, year, month, day1, day2)`` tuples, including + month-before and month-after date orders and the hyphenated instrument. +#. ``NTT EFOSC2 allocation 9-13 July`` creates **5** events; ``Magellan IMACS + 13-19 July`` creates **7**; ``Magellan Proto-Lightspeed Jul 8-12`` creates + **5** (``E - S + 1``, inclusive). +#. Each created event has ``start_time`` = dip-corrected sunset of its evening + date and ``end_time`` = sunrise of the following morning, both timezone-aware + UTC, with ``end_time > start_time`` and duration between 8 and 15 hours. +#. ``telescope`` and ``instrument`` are set from the line; the -15 deg dark + window and the original line appear in ``description``. +#. Running the command **twice** on the same input does not duplicate events + (idempotent). +#. A line whose run starts in late December creates events in the following + calendar year. + +*Stage 3 — queue window banner (#1)* + +#. An FTS UTC window creates exactly **one** ``CalendarEvent`` with + ``start_time`` / ``end_time`` equal to the supplied UTC bounds (no offset + applied), ``telescope='FTS'``, ``instrument='MuSCAT4'``. +#. The banner spans multiple dates (renders as an all-day banner) and records the + per-night hours and Moon constraint in ``description``. + +*Stage 4 — observation-record sync (#3)* + +#. Given an ``ObservationRecord`` (``facility='LCO'``) with ``scheduled_start`` / + ``scheduled_end`` set, the sync creates **one** ``CalendarEvent`` at those + exact times with ``url`` = the portal request URL. +#. Re-running the sync after the record's ``status`` changes updates the existing + event (matched by ``url``) rather than creating a second one. +#. A record in a terminal-failure state (e.g. ``WINDOW_EXPIRED``) results in its + event being removed (or marked), per the chosen policy. +#. Records without scheduled times (``PENDING`` only) create no block event. + +*End-to-end / manual verification* + +#. After ingesting the three sample classical runs plus one FTS window, the + ``/calendar/`` July 2026 view shows the La Silla and Las Campanas nights and + the FTS queue banner; opening an event shows the correct telescope, + instrument, UTC times and dark window. +#. The full suite passes: ``./manage.py test solsys_code`` and ``python -m + pytest`` both green; ``ruff check .`` and ``ruff format --check .`` clean. + +Open Items +---------- + +* The **FTS queue run input format** (#1) is not yet fixed; an example line of + how a UTC window + hours + Moon constraint is recorded is needed before + Stage 3. +* Terminal-failure policy for #3 (delete vs strike-through) to be confirmed. +* Whether ``Magellan`` should distinguish Baade vs Clay in ``telescope`` (the + ephemeris is identical; both are at Las Campanas). +* The Stage 1 ``SITES`` dict hardcodes telescope name -> MPC obscode, so + adding a new telescope requires a code change. Consider replacing it with a + lookup by ``Observatory.short_name`` directly (data-driven, no code change + to add sites) in Stage 2+. Also note ``to_earth_location()``/``sun_event()`` + assume a ground-based site; a guard against space-based observatories + (``Observatory.SATELLITE_OBSTYPE``, e.g. JWST/274) would be needed if + ``SITES`` (or its replacement) is ever extended to non-ground sites. diff --git a/docs/design/tom_calendar_vs_yse_pz_calendar.rst b/docs/design/tom_calendar_vs_yse_pz_calendar.rst new file mode 100644 index 00000000..5f68061f --- /dev/null +++ b/docs/design/tom_calendar_vs_yse_pz_calendar.rst @@ -0,0 +1,175 @@ +``tom_calendar`` vs YSE_PZ Calendar Support +============================================ + +This note compares the calendar support already used by FOMO +(:doc:`telescope_runs_calendar`, built on the TOM Toolkit's ``tom_calendar`` +package) against the calendar views in `YSE_PZ +`_ (Young Supernova Experiment - +PhotoMetry and Spectroscopy), a sibling time-domain follow-up TOM. Both +projects solve the same underlying problem -- show telescope time on a +calendar -- but with architectures different enough to be worth recording +before deciding how Stages 2-4 of issue #37 should evolve. + +Sources: ``tom_calendar`` is installed in the FOMO virtualenv +(``site-packages/tom_calendar``); YSE_PZ was read from a local clone +(``~/git/YSE_PZ``, ``YSE_App/models/``, ``YSE_App/views.py``, +``YSE_App/templates/YSE_App/*calendar*.html``). + +At a Glance +----------- + +.. list-table:: + :header-rows: 1 + :widths: 22 39 39 + + * - + - ``tom_calendar`` + - YSE_PZ + * - Data model + - One generic ``CalendarEvent`` model (+ ``EventTodo``), independent of + any domain model + - No calendar-specific model at all; each calendar view queries + existing domain models directly (``OnCallDate``, + ``ClassicalObservingDate``, ``ToOResource``, ``SurveyObservation``) + * - Number of calendars + - One generic calendar, reusable for any event type + - Five separate, purpose-built calendar pages (on-call, classical + observing, ToO, PS1/ZTF survey, DECam survey) + * - Create / edit / delete + - Inline, via htmx modals on the calendar grid itself + (``create_event`` / ``update_event`` / ``delete_event``) + - Not on the grid. On-call dates go through a separate + ``add_oncall_observer`` form view or the DRF API; the other four + calendars are read-only renders of data created elsewhere (proposals, + survey scheduler, ToO allocation) + * - Rendering + - Server-renders a month grid in Django; htmx swaps partials in place + (no full page reloads, no calendar JS library) + - Client-side `FullCalendar.js `_ (an old + ~2015-era jQuery build vendored via Bower); Django only emits a + JS ``events: [...]`` literal embedded in the template + * - Astronomy content + - None built in; moon phase is the only astronomy ``tom_calendar`` + itself computes (``MoonPhase.from_date``, used for the grid icon) + - Each observing calendar computes sunset/sunrise itself per view + (``astroplan.Observer.sun_set_time`` / ``sun_rise_time``), duplicated + across ``too_requests``, ``yse_home``, ``yse_observing_calendar``, + ``decam_observing_calendar`` + * - Multi-telescope handling + - Free-text ``telescope`` / ``instrument`` ``CharField`` per event; no + enforced relationship to a site model + - A real ``Telescope`` model with lat/lon/elevation; resources + (``ToOResource``, ``ClassicalResource``) hold an FK to it, so + coordinates are looked up, not duplicated + * - "Color" semantics + - ``CalendarEvent.color`` is a read-only property derived from ``pk`` + -- cannot be used to encode status + - Colors are computed per-view in Python (cycling through a fixed + palette, keyed by user or telescope) and passed into the JS event + objects, so color *can* encode meaning (e.g. one color per telescope) + +The Data Model +--------------- + +``tom_calendar.models.CalendarEvent`` is a single, domain-agnostic table: +``title``, ``description``, ``start_time``, ``end_time``, ``url``, +``telescope``, ``instrument``, ``proposal``, ``user`` (free text, not an FK), +an optional ``target_list`` FK, and a related ``EventTodo`` checklist. It +carries no opinion about *what kind* of event it represents -- an on-call +shift, a classical night, and a queue window would all be rows in the same +table, distinguished only by their field values. This is exactly what +:doc:`telescope_runs_calendar` exploits: Stage 1-4 reuse the existing fields +with no migration. + +YSE_PZ instead has dedicated models per concept, with real foreign keys: + +* ``OnCallDate`` / ``YSEOnCallDate`` -- a date plus a ``ManyToManyField`` to + ``User``. +* ``TelescopeResource`` (abstract) -- FK to ``Telescope``, optional FK to + ``PrincipalInvestigator``, ``begin_date_valid`` / ``end_date_valid`` + (the "semester"). Subclassed by ``ToOResource`` (adds awarded/used ToO + hours and triggers), ``QueuedResource`` (awarded/used hours), and + ``ClassicalResource`` (adds nothing beyond the base). +* ``ClassicalObservingDate`` -- FK to ``ClassicalResource`` and to + ``ClassicalNightType``, plus a single ``obs_date`` (one row **per night**, + not a date range). +* No model represents a "calendar event" in the abstract; the survey + calendars (``yse_observing_calendar``, ``decam_observing_calendar``) are + built directly from ``SurveyObservation`` rows that already exist for + scheduling purposes, with no calendar-specific persistence at all. + +The tradeoff: ``tom_calendar``'s generic model is reusable with zero schema +changes (good for FOMO's incremental Stage 1-4 plan) but pushes all +telescope/instrument/PI structure into untyped ``CharField`` text. YSE_PZ's +typed models give referential integrity (a ``Telescope`` has one +authoritative lat/lon/elevation; a ``ClassicalResource`` really does belong +to one PI) at the cost of a calendar view per concept and no generic +create/edit UI. + +The Views +--------- + +``tom_calendar.views`` (summarized fully in the prior chat turn) is six +small functions behind one ``app_name='tom_calendar'`` URL namespace: +``render_calendar`` (month grid, htmx partial or full page), and +``create_event`` / ``update_event`` / ``delete_event`` / ``create_todo`` / +``update_todo``, all of which mutate ``CalendarEvent``/``EventTodo`` and +re-render either a form partial or the calendar itself, firing htmx custom +events (``calRefresh``, ``calClose``) to update the page without a reload. +There is exactly one ``EventForm`` (a ``ModelForm`` on ``CalendarEvent``) +covering every event, classical or queue alike. + +YSE_PZ's calendar views (``YSE_App/views.py``) are five separate, +hand-written, read-only renders, each tied to one model and visualization: + +* ``calendar`` / ``yse_oncall_calendar`` -- on-call rosters, colored per + user, built from ``OnCallDate`` / ``YSEOnCallDate``. +* ``observing_calendar`` -- classical nights, colored per telescope, from + ``ClassicalObservingDate``. +* ``too_calendar`` -- a 60-days-back/60-days-forward window of + ``ToOResource`` validity ranges, with per-day date lists computed in + Python and handed to the template. +* ``yse_observing_calendar`` / ``decam_observing_calendar`` -- a hardcoded + 40-day window (today - 30 to +9) of ``SurveyObservation`` rows for PS1/ZTF + or DECam, with sunset/sunrise and moon illumination recomputed per day + inside the view, and pre-formatted summary strings built with manual + string concatenation rather than template logic. + +None of the YSE_PZ calendar views accept a month/year query parameter the +way ``render_calendar`` does -- each hardcodes its own date window in Python +and is not navigable forward/backward. Creating new entries happens outside +the calendar entirely (a separate form view for on-call dates, the Django +admin or DRF API for the resource models). + +What This Means for Issue #37 +------------------------------ + +FOMO's existing plan (:doc:`telescope_runs_calendar`) already chose the +``tom_calendar`` approach -- reuse the generic ``CalendarEvent`` model, +synced/ingested by management commands -- and this comparison does not +surface anything that would change that choice for Stages 2-4: + +* FOMO has one generic need (telescope runs as calendar blocks), not five + visually distinct calendar products, so ``tom_calendar``'s single reusable + model is the better fit; replicating YSE_PZ's "one bespoke view per + data source" pattern would mean writing (and maintaining) a separate + calendar per telescope/scheduling-model combination. +* ``tom_calendar``'s htmx create/edit/delete-on-the-grid UI is more capable + than anything in YSE_PZ's calendars (which have no inline editing at all), + so Stage 2's idempotent management-command ingest is additive on top of + an already-richer UI, not a gap to fill. +* YSE_PZ's pattern of recomputing sunset/sunrise/twilight independently in + four different views (``too_requests``, ``yse_home``, + ``yse_observing_calendar``, ``decam_observing_calendar``) is the duplication + Stage 1's shared ``solsys_code/telescope_runs.py`` helper is explicitly + designed to avoid -- worth treating as a cautionary example rather than a + pattern to copy. +* The one idea worth borrowing: YSE_PZ's ``Telescope`` model (FK'd from + every resource) instead of free-text fields. FOMO already has the + equivalent in ``Observatory`` (MPC-obscode keyed); Stage 1 deliberately + looks sites up from it rather than hardcoding coordinates, and the Open + Items in :doc:`telescope_runs_calendar` already flag making the + ``SITES`` registry data-driven from ``Observatory`` -- the same direction + YSE_PZ's FK-based design points to, just not yet applied to + ``CalendarEvent.telescope`` itself (which remains free text, as + ``tom_calendar`` defines it). diff --git a/docs/design/uncertain_scheduling_spike.rst b/docs/design/uncertain_scheduling_spike.rst new file mode 100644 index 00000000..d9d68bef --- /dev/null +++ b/docs/design/uncertain_scheduling_spike.rst @@ -0,0 +1,111 @@ +Uncertain-Scheduling Investigation Spike +======================================== + +This document records the investigation spike that settled five open design +decisions for FOMO's ``CampaignRun`` scheduling model against the real +3I/ATLAS coordination sheet (2026-07-09 snapshot). It was written after a +live investigation that read the actual, publicly-editable Google Sheet +export and probed the live local ``Observatory`` DB and MPC Obscodes API, +rather than reasoning from synthetic examples or documentation alone. No +``CampaignRun`` schema migration, no CSV importer change, and no fuzzy-match +UI code was built during this spike — the deliverable is this durable +summary and its full-detail companion, ``18-DECISION.md`` (originally at +``.planning/phases/18-uncertain-scheduling-investigation-spike/18-DECISION.md``; +this project's milestone-archival workflow moves completed phase directories +to ``.planning/phases-archive/`` once their milestone closes, so check there +first if the original path no longer resolves). + +Background +---------- + +FOMO's ``CampaignRun`` model coordinates community-submitted observation +plans for 3I/ATLAS, imported from a real, live, publicly-editable Google +Sheet. The real sheet's space-mission rows (JWST, HST, Swift) frequently +carry a date *range* or a still-``TBD`` observing date rather than a single +known night — the sheet's actual rows are messier than the synthetic +``campaign_sample.csv`` fixture anticipated. Before Phase 19 migrates the +schema, Phase 20 extends the CSV importer, and Phase 21 builds a +staff-facing fuzzy-match site-resolution UI, this spike settled five +concrete questions against the real sheet and the live ``Observatory`` data +so none of that downstream work has to re-derive them from scratch: + +* The window field schema for representing a range/TBD observing date. +* The replacement natural key for rows sharing a still-unknown observing + date. +* The CSV range/TBD text-parsing rules the importer needs. +* The fuzzy-match library choice for staff-facing site-code resolution. +* Whether ``resolve_site()`` correctly resolves real space-observatory MPC + codes, and whether ``Observatory.obscode`` needs widening. + +Key finding +----------- + +**Window schema: confirmed as the already-locked nullable +``window_start``/``window_end`` ``DateField`` pair — no schema change needed.** +Every real cell shape in the sheet (single exact date, full-date range, +compact same-month range, and the day-unknown ``TBD`` marker) maps cleanly +onto this pair. + +**Fuzzy-match library: difflib is the primary choice — no new dependency +justified by this live test.** ``rapidfuzz`` and stdlib ``difflib`` produced the +same matches (including the same two false positives) and the same clean +misses on the real messy ``Site Code`` corpus; add ``rapidfuzz`` to +``pyproject.toml`` explicitly only if a future, wider candidate pool +demonstrates a case difflib genuinely misses. + +Decisions +--------- + +.. list-table:: + :header-rows: 1 + :widths: 22 48 12 + + * - SCHED-01 criterion + - Decision + - Phase + * - Window field schema + - Nullable ``window_start``/``window_end`` ``DateField`` pair, confirmed + against real single-date, ranged, and TBD cell shapes. + - 19 + * - TBD-row natural key + - Fold ``contact_person`` (existing ``CampaignRun`` field) into the + natural key for rows where ``window_start IS NULL``, via a + partial/conditional ``UniqueConstraint`` (exact mechanism is Phase + 19's to design) — evidenced by a real two-row JWST collision in the + live sheet. + - 19 + * - CSV range/TBD parsing rules + - Extend ``parse_obs_window()``'s existing pattern-per-shape discipline + (the same approach already used for ``_HHMM_RANGE``, ``_APPROX_HOUR``, + and ``_BARE_HOUR_UTC``) to ``Obs. Date``, one rule per real shape; + range-detection must inspect both ``Obs. Date`` and + ``UT Time Range``; never raise on messy non-key fields, flag + needs-review instead. + - 20 + * - Fuzzy-match library + - ``difflib.get_close_matches`` as the primary/default choice; add + ``rapidfuzz`` to ``pyproject.toml`` explicitly only if a later, wider + candidate pool proves it necessary. + - 21 + * - ``resolve_site()`` / obscode widening + - No widening of ``Observatory.obscode`` (``max_length=4``) needed — + confirmed against the live field definition; real space-observatory + MPC codes (``250``, ``274``, ``289``) all fit within 3 characters. + Separately, ``resolve_site()`` cannot *currently* resolve any of the + three: ``MPCObscodeFetcher.to_observatory()`` raises an unguarded + ``TypeError`` on the MPC API's ``null`` longitude for satellite-type + records, an unrelated bug for Phase 19/21 to be aware of (see + Future scope). + - None (confirmed as-is) + +Future scope +------------ + +See ``18-DECISION.md`` (path note above) for the full evidence each of these +decisions rests on — including the live rapidfuzz/difflib score comparison +against the real messy ``Site Code`` corpus, the real JWST natural-key +collision, the enumerated CSV cell shapes from the live 2026-07-09 sheet +snapshot, and the unrelated ``to_observatory()`` ``TypeError`` on +satellite-type MPC records discovered while confirming the obscode-widening +verdict. These are recommendations for Phases 19-21 to implement, not +implemented in this spike. diff --git a/docs/notebooks/ESO_How_to_download_data.ipynb b/docs/notebooks/ESO_How_to_download_data.ipynb new file mode 100644 index 00000000..ee75195c --- /dev/null +++ b/docs/notebooks/ESO_How_to_download_data.ipynb @@ -0,0 +1,592 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "ESO Logo \n", + "#  How to download data \n", + "
\n", + "\n", + "This section of the [\"ESO Science Archive Programmatic: HOWTOs\"](http://archive.eso.org/programmatic/HOWTO/) shows how to programmatically download ESO data, either anonymously (for public data) or with authentication (for proprietary data), using Python.\n", + "\n", + "_**Usage**: You can access this file either as a static HTML page [(download it here)](http://archive.eso.org/programmatic/HOWTO/jupyter/ESO_How_to_download_data.html), or as an interactive jupyter notebook [(download it here)](http://archive.eso.org/programmatic/HOWTO/jupyter/ESO_How_to_download_data.ipynb) which you can download and run on your machine [(instructions)](https://jupyter.org/install). To interact with the jupyter notebook: move up and down the various cells using the arrow keys, execute the code by pressing CTRL+ENTER; you can also modify the code and execute it at will._\n", + "\n", + "
\n", + "\n", + "Let's start by setting up the python modules:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "\n", + "import requests\n", + "import cgi\n", + "import json\n", + "\n", + "import getpass" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's define a couple of utility functions, useful to write the files on disk using the ESO file name (provided in the response http header, via the Content-Disposition field." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def getDispositionFilename(response):\n", + " \"\"\"Get the filename from the Content-Disposition in the response's http header\"\"\"\n", + " contentdisposition = response.headers.get('Content-Disposition')\n", + " if contentdisposition == None:\n", + " return None\n", + " value, params = cgi.parse_header(contentdisposition)\n", + " filename = params['filename']\n", + " return filename\n", + "\n", + "\n", + "def writeFile(response, dirname='data'):\n", + " \"\"\"Write on disk the retrieved file\"\"\"\n", + " if response.status_code == 200:\n", + " # The ESO filename can be found in the response header\n", + " filename = getDispositionFilename(response)\n", + " os.makedirs(dirname, exist_ok=True)\n", + " filepath = os.path.join(dirname, filename)\n", + " # Let's write on disk the downloaded FITS spectrum using the ESO filename:\n", + " with open(filepath, 'wb') as f:\n", + " f.write(response.content)\n", + " return filepath" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## How to retrieve a file anonymously\n", + "\n", + "Without the need to authenticate, any user can anonymously download public files, that is, files that are out of the proprietary period (of usually one year from the moment the observation takes place)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "file_url = 'https://dataportal.eso.org/dataportal_new/file/ADP.2016-11-17T12:51:01.877'\n", + "\n", + "response = requests.get(file_url)\n", + "filename = writeFile(response)\n", + "if filename:\n", + " print('Saved file: %s' % (filename))\n", + "else:\n", + " print('Could not get file (status: %d)' % (response.status_code))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## How to retrieve a file with authentication\n", + "\n", + "If the files you need to retrieve are under proprietary period, you can access them only if you have the rights to do so, that is, if you are the principal investigator [PI] of the observing program the files belong to, or one of his/her delegates. In this case, you certainly got already a (free) user account at the [ESO User Portal](https://www.eso.org/UserPortal).\n", + "\n", + "Before downloading the file you have to authenticate and get a token. Here is the method that, given your ESO credentials (username and password), returns the token." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def getToken(username, password):\n", + " \"\"\"Token based authentication to ESO: provide username and password to receive back a JSON Web Token.\"\"\"\n", + " if username == None or password == None:\n", + " return None\n", + " token_url = 'https://www.eso.org/sso/oidc/token'\n", + " token = None\n", + " try:\n", + " response = requests.get(\n", + " token_url,\n", + " params={\n", + " 'response_type': 'id_token token',\n", + " 'grant_type': 'password',\n", + " 'client_id': 'clientid',\n", + " 'username': username,\n", + " 'password': password,\n", + " },\n", + " )\n", + " token_response = json.loads(response.content)\n", + " token = token_response['id_token'] + '=='\n", + " except NameError as e:\n", + " print(e)\n", + " except:\n", + " print('*** AUTHENTICATION ERROR: Invalid credentials provided for username %s' % (username))\n", + "\n", + " return token" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Once you get the token, you need to add it to the HTTP header before HTTP-getting the file. Let's see how:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Suppose the file you want to download is accessible via the following link\n", + "# (please change the identifier (ADP.2020-03-24T1:45:21.866) to one of your proprietary files):\n", + "\n", + "file_url = 'https://dataportal.eso.org/dataportal_new/file/ADP.2020-03-24T10:45:21.866'\n", + "\n", + "# If you have not modified the identifier of that file,\n", + "# likely you won't be authorised to download the file.\n", + "# Expect a failure \"Could not get file (status: 401)\" in that case.\n", + "\n", + "# Let's get the token, by inputting your credentials:\n", + "username = input('Type your ESO username: ')\n", + "password = getpass.getpass(prompt=\"%s user's password: \" % (username), stream=None)\n", + "token = getToken(username, password)\n", + "\n", + "# With successful authentication you get a valid token,\n", + "# which needs to be added to the HTTP header of your GET request,\n", + "# as a Bearer:\n", + "\n", + "headers = None\n", + "if token != None:\n", + " headers = {'Authorization': 'Bearer ' + token}\n", + " response = requests.get(file_url, headers=headers)\n", + " filename = writeFile(response)\n", + " if filename:\n", + " print('Saved file: %s' % (filename))\n", + " else:\n", + " print('Could not get file (status: %d)' % (response.status_code))\n", + "else:\n", + " print('Could not authenticate')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pyvo\n", + "from pyvo.dal import tap\n", + "from pyvo.auth.authsession import AuthSession" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import eso_programmatic as eso\n", + "\n", + "TAP_URL = 'http://archive.eso.org/tap_obs'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Prompt for user's credentials and get a token\n", + "import getpass\n", + "\n", + "username = input('Type your ESO username: ')\n", + "password = getpass.getpass(prompt=f\"{username}'s password: \", stream=None)\n", + "\n", + "token = eso.getToken(username, password)\n", + "if token != None:\n", + " print('token: ' + token)\n", + "else:\n", + " sys.exit(-1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "session = requests.Session()\n", + "session.headers['Authorization'] = 'Bearer ' + token\n", + "\n", + "# Initialise a tap service for authorised queries\n", + "# passing the created \"tokenised\" session\n", + "# Remember: passing a non tokenised-session, or no session at all,\n", + "# will result in tap performing anonymous queries:\n", + "# none of your permissions will be used, hence the queryies will run faster,\n", + "# and you will not be able to find any file with protected metadata.\n", + "\n", + "tap = pyvo.dal.TAPService(TAP_URL, session=session)\n", + "\n", + "# for comparison, use:\n", + "# tap = pyvo.dal.TAPService(TAP_URL)\n", + "# to execute your queries anonymously" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# define the query you want to run, e.g.:\n", + "query = \"select top 2 * from dbo.raw where dp_cat='SCIENCE' and prog_id = 'your-protected-observing-run' \"\n", + "\n", + "# well, in this example we use a non-protected run,\n", + "# but please pretend it is actually a protected one given the purpose of this notebook!\n", + "\n", + "# let's consider only 2 of its science frames:\n", + "query = \"select top 10 * from dbo.raw where dp_cat='SCIENCE' and prog_id = '116.28N5.001' \"\n", + "\n", + "\n", + "results = None\n", + "\n", + "# define a job that will run the query asynchronously\n", + "job = tap.submit_job(query)\n", + "\n", + "# extending the maximum duration of the job to 300s (default 60 seconds)\n", + "job.execution_duration = 300 # max allowed: 3600s\n", + "\n", + "# job initially is in phase PENDING; you need to run it and wait for completion:\n", + "job.run()\n", + "\n", + "try:\n", + " job.wait(phases=['COMPLETED', 'ERROR', 'ABORTED'], timeout=600.0)\n", + "except pyvo.DALServiceError:\n", + " print('Exception on JOB {id}: {status}'.format(id=job.job_id, status=job.phase))\n", + "\n", + "print('Job: %s %s' % (job.job_id, job.phase))\n", + "\n", + "if job.phase == 'COMPLETED':\n", + " # When the job has completed, the results can be fetched:\n", + " results = job.fetch_result()\n", + "\n", + "# the job can be deleted (always a good practice to release the disk space on the ESO servers)\n", + "job.delete()\n", + "\n", + "# Let's print the results to examine the content:\n", + "# check out the access_url and the datalink_url\n", + "if results:\n", + " print('query results:')\n", + " eso.printTableTransposedByTheRecord(results.to_table())\n", + "else:\n", + " print('!' * 42)\n", + " print('! !')\n", + " print('! No results could be found. !')\n", + " print('! ? Perhaps no permissions ? !')\n", + " print('! Aborting here. !')\n", + " print('! !')\n", + " print('!' * 42)\n", + " quit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3 Downloading the selected science files using their access_url" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The access_url field of the dbo.raw table\n", + "# provides the link that can be used to download the file\n", + "\n", + "# Here we pass that link together with your session\n", + "# to the downloadURL method of the eso_programmatic.py module\n", + "# (similarly to the authorised queries, if no session is passed,\n", + "# downloadURL will attempt to download the file anonymously)\n", + "\n", + "print('Start downloading...')\n", + "for raw in results:\n", + " access_url = raw['access_url'] # the access_url is the link to the raw file\n", + " status, filepath = eso.downloadURL(access_url, session=session, dirname='/apophis/tlister/VLT/FORS/116.28N5.001/')\n", + " if status == 200:\n", + " print(f' RAW: {filepath} downloaded ')\n", + " else:\n", + " print('ERROR RAW: {filepath} NOT DOWNLOADED (http status:{status})')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4 Finding and downloading the associated calibration reference files" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.1 Find the link to the associated calibration reference files (using DataLink)\n", + "\n", + "The datalink_url field of the dbo.raw table provides you the link that can be used to find files associated to the selected science frame." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# A python datalink object is created running\n", + "# the pyvo DataLinkResults.from_result_url() method onto the datalink_url.\n", + "\n", + "# When dealing with files whose metadata are protected, we need to be authorised:\n", + "# for that we need to pass to the from_result_url() also the above-created python requests session.\n", + "\n", + "# For the sake of this example, let's just consider the first science raw frame:\n", + "first_record = results[0]\n", + "datalink_url = first_record['datalink_url']\n", + "\n", + "datalink = pyvo.dal.adhoc.DatalinkResults.from_result_url(datalink_url, session=session)\n", + "\n", + "# The resulting datalink object contains the table of files associated\n", + "# to SPHER.2016-09-26T03:04:09.308\n", + "# Note: Were this input file a metadata protected file (it is not, but suppose...),\n", + "# and had you not passed your session, or had you no permission to see this file,\n", + "# DataLink would have given you back only a laconic table with the message\n", + "# that that you do not have access permissions or that the file does not exist.\n", + "\n", + "# let's print the resulting datalink table:\n", + "eso.printTableTransposedByTheRecord(datalink.to_table())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Let's get the link to the processed calibration files (raw2master)\n", + "\n", + "semantics = 'http://archive.eso.org/rdf/datalink/eso#calSelector_raw2master'\n", + "\n", + "raw2master_url = next(datalink.bysemantics(semantics)).access_url\n", + "\n", + "# which returns the calSelector (see next box) link:\n", + "# https://archive.eso.org/calselector/v1/associations?dp_id=\\\n", + "# SPHER.2016-09-26T03:04:09.308&mode=Raw2Master&responseformat=votable" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Don't forget to pass your session in case the science file has protected metadata!\n", + "\n", + "associated_calib_files = pyvo.dal.adhoc.DatalinkResults.from_result_url(raw2master_url, session=session)\n", + "\n", + "eso.printTableTransposedByTheRecord(associated_calib_files.to_table())\n", + "\n", + "# create and use a mask to get only the #calibration entries,\n", + "# given that other entries, like #this or ...#sibiling_raw, could be present:\n", + "calibrator_mask = associated_calib_files['semantics'] == '#calibration'\n", + "calib_urls = associated_calib_files.to_table()[calibrator_mask]['access_url', 'eso_category']\n", + "\n", + "# eso.printTableTransposedByTheRecord(calib_urls)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.2 Getting the list of processed calibration reference files (using calSelector and DataLink)\n", + "The automatic selection of calibration files (raw or processed) is performed by the above-mentioned calSelector service, exposed also programmatically.\n", + "\n", + "One of the calSelector interfaces (the responseformat=votable param must be present), is fully compatible with the datalink VO protocol. This means that the same pyvo DatalinkResults.from_result_url() method can be used, e.g., to get the list of associated raw2master files." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Don't forget to pass your session in case the science file has protected metadata!\n", + "\n", + "associated_calib_files = pyvo.dal.adhoc.DatalinkResults.from_result_url(raw2master_url, session=session)\n", + "\n", + "eso.printTableTransposedByTheRecord(associated_calib_files.to_table())\n", + "\n", + "# create and use a mask to get only the #calibration entries,\n", + "# given that other entries, like #this or ...#sibiling_raw, could be present:\n", + "calibrator_mask = associated_calib_files['semantics'] == '#calibration'\n", + "calib_urls = associated_calib_files.to_table()[calibrator_mask]['access_url', 'eso_category']\n", + "\n", + "# eso.printTableTransposedByTheRecord(calib_urls)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.2.1 Check calibration cascade qualities\n", + "\n", + "Check if calibration cascade is complete, if it is certified, and if it is actually for processed calib files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Given the above list of \"associated_calib_files\"\n", + "# and knowing that we requested...\n", + "mode_requested = 'raw2master'\n", + "\n", + "# ... let's print out some important info and warnings on the received calibration cascade:\n", + "# - is the cascade complete?\n", + "# - is the cascade certified?\n", + "# - has the cascade being generated for the mode you requested (processed calibrations) or not?\n", + "\n", + "# That info is embedded in the description field of the #this record.\n", + "# We use the printCalselectorInfo of the eso_programmatic.py to parse/make sense of it.\n", + "\n", + "this_description = next(associated_calib_files.bysemantics('#this')).description\n", + "\n", + "alert, mode_warning, certified_warning = eso.printCalselectorInfo(this_description, mode_requested)\n", + "\n", + "if alert != '':\n", + " print('%s' % (alert))\n", + "if mode_warning != '':\n", + " print('%s' % (mode_warning))\n", + "if certified_warning != '':\n", + " print('%s' % (certified_warning))\n", + "\n", + "question = None\n", + "answer = None\n", + "if len(calib_urls):\n", + " print()\n", + " if alert or mode_warning or certified_warning:\n", + " question = 'Given the above warning(s), do you still want to download these %d calib files [y/n]? ' % (\n", + " len(calib_urls)\n", + " )\n", + " else:\n", + " question = 'No warnings reported, do you want to download these %d calib files [y/n]? ' % (len(calib_urls))\n", + "\n", + "while answer != 'y' and answer != 'n':\n", + " answer = input(question)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4.3 Downloading the calibration reference files\n", + "\n", + "To download the calibration files we use again the downloadURL method of the eso_programmatic.py module.\n", + "\n", + "All ESO calibration files are open to the public, hence there is no need to pass your token/session.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if answer == 'y':\n", + " print('Downloading the %d calibration reference files...' % (len(calib_urls)))\n", + "\n", + " i_calib = 0\n", + " for url, category in calib_urls:\n", + " i_calib += 1\n", + " status, filename = eso.downloadURL(url, dirname='/apophis/tlister/VLT/FORS/116.28N5.001/')\n", + " if status == 200:\n", + " print(' CALIB: %4d/%d dp_id: %s (%s) downloaded' % (i_calib, len(calib_urls), filename, category))\n", + " else:\n", + " print(\n", + " ' CALIB: %4d/%d dp_id: %s (%s) NOT DOWNLOADED (http status:%d)'\n", + " % (i_calib, len(calib_urls), filename, category, status)\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "association_tree_semantics = 'http://archive.eso.org/rdf/datalink/eso#calSelector_raw2master'\n", + "\n", + "# Notice that the datalink service and the calselector service use the same semantics\n", + "# to indicate two different things:\n", + "# - in datalink: it points to the distinct list of calibration reference files (responseformat=votable);\n", + "# its eso_category is not defined\n", + "# - in calselector: it points to the calibration cascade description (format still XML but not votable);\n", + "# its eso_category is set to \"ASSOCIATION_TREE\"\n", + "\n", + "association_tree_mask = associated_calib_files['semantics'] == association_tree_semantics\n", + "association_tree = associated_calib_files.to_table()[association_tree_mask]['access_url', 'eso_category']\n", + "\n", + "for url, category in association_tree:\n", + " # the url points to the calselector service, which, for metadata protected files, needs a tokenised-session\n", + " status, filename = eso.downloadURL(url, session=session)\n", + " print(url)\n", + " if status == 200:\n", + " print(' Association tree: %s (%s) downloaded' % (filename, category))\n", + " else:\n", + " print(' Association tree: %s (%s) NOT DOWNLOADED (http status:%d)' % (filename, category, status))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.13" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/notebooks/pre_executed/fixtures/campaign_sample.csv b/docs/notebooks/pre_executed/fixtures/campaign_sample.csv new file mode 100644 index 00000000..6482c190 --- /dev/null +++ b/docs/notebooks/pre_executed/fixtures/campaign_sample.csv @@ -0,0 +1,9 @@ +Contact Person,Email,Telescope / Instrument,Site Code,Obs. Date,UT Time Range,Filter(s)/Bandpass,Observation Details,Weather conditions or forecast,Observation Status,Observation Outcome,Publication Plans,Open to collaboration?,Other comments +Ada Test,ada.test@example.com,FTN/MuSCAT3,F65,2025-07-04,08:50 - 11:50,griz,Simultaneous 4-band imaging of nucleus and near-nucleus coma,Clear; seeing 1.0 arcsec,completed,Clean detection in all 4 bands; no obvious tail asymmetry,TBD; pending team discussion,no, +Ben Sample,ben.sample@example.org,ESO VLT FORS2,309,2025-07-04,06:50 - 07:15,R,Long-slit optical spectrum R~440,Thin cirrus,completed,Spectrum shows strong OH emission band,Yes -- spectroscopy paper in prep,yes, +Cy Fixture,cy.fixture@example.com,Apache Point Observatory/ARCTIC,705,2025-07-06,05:30 - 06:00,g/r/i/z,Broadband imaging attempt aborted mid-sequence,High clouds moved in,cancelled - weather,,,no,Reschedule requested for next lunation +Dee Approx,dee.approx@example.com,VLT/MUSE,309,2025-07-16,~1 am,480-930 nm,IFU monitoring cadence single exposure,Photometric,completed,Nucleus well resolved from coma,Survey paper; co-authorship open,no, +Eli Blank,eli.blank@example.com,Apache Point Observatory/KOSMOS,705,2025-07-06,,Blue GRISM 3800-6600 A,Low-res spectroscopy exact start time not logged,Good,completed,Detected continuum; weak emission features,Open to shared authorship,yes,Contact for raw data access +Fay Review,fay.review@example.com,Generic 1m robotic telescope,,2025-07-11,09:00 - 09:30,V,Rapid-response photometry site not yet confirmed,Clear,Upcoming,,,no,Site TBD needs manual review +Gia Range,gia.range@example.com,FTN/FLOYDS,F65,2025-08-01 to 2025-08-15,,BVRI,Multi-night monitoring window not yet narrowed to a single night,Variable,Upcoming,,,no,Window reserved pending scheduling +Ike Pending,ike.pending@example.com,VLT/X-shooter,309,TBD pending Cycle 2,,NUV-NIR,Awaiting Cycle 2 time allocation decision,,Upcoming,,,no,Contact Person required for a stable TBD natural key diff --git a/docs/notebooks/pre_executed/import_campaign_csv_demo.ipynb b/docs/notebooks/pre_executed/import_campaign_csv_demo.ipynb new file mode 100644 index 00000000..1e5425c2 --- /dev/null +++ b/docs/notebooks/pre_executed/import_campaign_csv_demo.ipynb @@ -0,0 +1,578 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6f00583f", + "metadata": {}, + "source": [ + "# Campaign CSV Bootstrap Import Demo\n", + "\n", + "This notebook demonstrates `solsys_code/management/commands/import_campaign_csv.py`\n", + "(Phase 14, CAMP-04/CAMP-05), the bootstrap-import command that turns a campaign\n", + "coordination CSV (e.g. the real 3I/ATLAS Google Sheet) into `CampaignRun` rows.\n", + "\n", + "**The fixture used here (`fixtures/campaign_sample.csv`) is entirely synthetic and\n", + "PII-free** (CAMP-05) — every contact name is a placeholder (`Ada Test`, `Ben Sample`, ...)\n", + "and every email address is `@example.com`/`@example.org`. No real contact information\n", + "from the actual 3I/ATLAS coordination sheet is ever read by this notebook.\n", + "\n", + "It demonstrates:\n", + "\n", + "- Seeding `Observatory` records for the 3 sites used by the fixture (idempotent\n", + " `update_or_create`), so site resolution hits the local database only — no live\n", + " MPC Obscodes API call happens anywhere in this notebook (D-11)\n", + "- Seeding a single-`Target` campaign `TargetList` so the auto-target-resolution\n", + " behavior (D-07/CAMP-02) is exercised\n", + "- Invoking `import_campaign_csv` via `call_command` and inspecting the printed\n", + " created/updated/unchanged/skipped/site_needs_review summary\n", + "- Inspecting the resulting `CampaignRun` rows\n", + "- The `pending_review` -> `approved`/`rejected` `approval_status` lifecycle (D-03)\n", + " on synthetic data — the bootstrap import itself always writes `approved`\n", + " (vetted historical backfill), so this lifecycle has no other demonstration path\n", + "- Re-running the command to confirm idempotency (no duplicate rows)\n", + "\n", + "This notebook lives in `pre_executed/` because it is **DB-dependent** (it seeds\n", + "`Observatory`/`TargetList` records and creates `CampaignRun` rows) and is therefore\n", + "**NOT run during Sphinx/CI/ReadTheDocs builds**, per `docs/notebooks/README.md`." + ] + }, + { + "cell_type": "markdown", + "id": "43e2c1c3", + "metadata": {}, + "source": [ + "## Django setup\n", + "\n", + "Standard boilerplate to make `src.fomo.settings` importable from this notebook's\n", + "location (`docs/notebooks/pre_executed/` — three levels under the repo root, so\n", + "`parents[2]` gives the repo root) and to allow synchronous ORM calls inside\n", + "Jupyter's async event loop." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "49da5beb", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-10T19:57:03.465775Z", + "iopub.status.busy": "2026-07-10T19:57:03.465453Z", + "iopub.status.idle": "2026-07-10T19:57:05.629177Z", + "shell.execute_reply": "2026-07-10T19:57:05.627221Z" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "import django\n", + "\n", + "# Ensure the repo root is on sys.path so `src.fomo.settings` is importable\n", + "# when this notebook is executed from docs/notebooks/pre_executed/.\n", + "# NOTE: parents[2] is correct only when the Jupyter kernel CWD is\n", + "# docs/notebooks/pre_executed/. Start Jupyter from that directory, or\n", + "# adjust the index if you launch from the repo root.\n", + "repo_root_path = Path.cwd().resolve().parents[2]\n", + "assert (\n", + " repo_root_path / 'manage.py'\n", + ").exists(), (\n", + " f'Repo root not found at {repo_root_path}. Run Jupyter from docs/notebooks/pre_executed/ or adjust parents[] index.'\n", + ")\n", + "repo_root = str(repo_root_path)\n", + "if repo_root not in sys.path:\n", + " sys.path.insert(0, repo_root)\n", + "\n", + "os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'src.fomo.settings')\n", + "\n", + "# Jupyter's ipykernel runs inside an asyncio event loop, but Django's ORM is\n", + "# sync-only by default and refuses to run there; this opts back in.\n", + "os.environ.setdefault('DJANGO_ALLOW_ASYNC_UNSAFE', 'true')\n", + "\n", + "django.setup()\n", + "\n", + "# This notebook intentionally imports only the campaign-coordination models and\n", + "# the observatory package below -- never the ephemeris view/computation modules,\n", + "# which trigger a large one-time SPICE kernel download on first import." + ] + }, + { + "cell_type": "markdown", + "id": "29132f16", + "metadata": {}, + "source": [ + "## Seed Observatory records and the campaign TargetList\n", + "\n", + "`import_campaign_csv` resolves each row's `Site Code` against the `Observatory`\n", + "model first (tier 1 of the D-08 3-tier resolution); only a tier-1 miss falls\n", + "through to a live MPC Obscodes API call. The fixture's three non-blank Site\n", + "Codes (`F65`, `309`, `705`) are seeded here first, so every row in this\n", + "notebook resolves locally and no network call happens (D-11). `update_or_create`\n", + "makes this cell idempotent — safe to re-run against any dev DB.\n", + "\n", + "The campaign container is a `tom_targets.models.TargetList` — found-or-created by\n", + "name (D-06). Linking exactly one `Target` to it (via `NonSiderealTargetFactory`,\n", + "per this project's Target-factory convention — FOMO is exclusively for Solar\n", + "System / non-sidereal targets) demonstrates auto-target resolution: every\n", + "imported `CampaignRun` gets that `Target` assigned automatically (D-07/CAMP-02)." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "465fe73b", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-10T19:57:05.633958Z", + "iopub.status.busy": "2026-07-10T19:57:05.633511Z", + "iopub.status.idle": "2026-07-10T19:57:05.856750Z", + "shell.execute_reply": "2026-07-10T19:57:05.854806Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " obscode='F65' updated/unchanged short_name='FTN'\n", + " obscode='309' updated/unchanged short_name='VLT'\n", + " obscode='705' updated/unchanged short_name='APO'\n", + "\n", + "Campaign TargetList '3I/ATLAS (demo)' found\n", + " Already has 1 linked Target(s)\n", + "Campaign now has 1 linked Target(s): ['3I/ATLAS (demo target)']\n" + ] + } + ], + "source": [ + "from tom_targets.models import TargetList\n", + "from tom_targets.tests.factories import NonSiderealTargetFactory\n", + "\n", + "from solsys_code.solsys_code_observatory.models import Observatory\n", + "\n", + "SITE_DATA = {\n", + " 'F65': dict(name='Faulkes Telescope North', short_name='FTN', lat=20.7, lon=-156.3, altitude=3055),\n", + " '309': dict(\n", + " name='European Southern Observatory, Paranal', short_name='VLT', lat=-24.6272, lon=-70.4039, altitude=2635\n", + " ),\n", + " '705': dict(name='Apache Point Observatory', short_name='APO', lat=32.7803, lon=-105.8203, altitude=2788),\n", + "}\n", + "\n", + "for obscode, fields in SITE_DATA.items():\n", + " obs, created = Observatory.objects.update_or_create(obscode=obscode, defaults=fields)\n", + " action = 'created' if created else 'updated/unchanged'\n", + " print(f' obscode={obscode!r:>4} {action:>18} short_name={obs.short_name!r}')\n", + "\n", + "campaign, campaign_created = TargetList.objects.get_or_create(name='3I/ATLAS (demo)')\n", + "print(f'\\nCampaign TargetList {campaign.name!r} {\"created\" if campaign_created else \"found\"}')\n", + "\n", + "if campaign.targets.count() == 0:\n", + " demo_target = NonSiderealTargetFactory.create(name='3I/ATLAS (demo target)')\n", + " campaign.targets.add(demo_target)\n", + " print(f' Linked new Target: {demo_target.name!r}')\n", + "else:\n", + " print(f' Already has {campaign.targets.count()} linked Target(s)')\n", + "\n", + "print(f'Campaign now has {campaign.targets.count()} linked Target(s): {[t.name for t in campaign.targets.all()]}')" + ] + }, + { + "cell_type": "markdown", + "id": "48b89ebf", + "metadata": {}, + "source": [ + "## The synthetic fixture\n", + "\n", + "`fixtures/campaign_sample.csv` has the same 14-column shape as the real\n", + "3I/ATLAS coordination sheet, but every row is hand-built synthetic data (D-10):\n", + "placeholder contact names, `@example.com`/`@example.org` emails, and only the\n", + "three `Site Code` values seeded above. It covers:\n", + "\n", + "- a clean multi-band imaging row (`griz`, a normal `HH:MM - HH:MM` UT range)\n", + "- a spectroscopy row (`Open to collaboration? = yes`)\n", + "- an `Observation Status` that maps to a terminal `run_status` (`cancelled`)\n", + "- an approximate UT time (`~1 am`) and a fully blank UT time, exercising both\n", + " of `parse_obs_window`'s best-effort fallback paths\n", + "- a blank `Site Code`, exercising the `site_needs_review` flag with no\n", + " Observatory match attempted\n", + "- a date-range `Obs. Date` (`'2025-08-01 to 2025-08-15'`), exercising D-12's full-date\n", + " range parsing (IMPORT-01)\n", + "- a genuinely-unparseable `Obs. Date` (`'TBD pending Cycle 2'`) with a non-blank\n", + " `Contact Person`, exercising D-13's never-raise TBD contract (IMPORT-02)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "1093927e", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-10T19:57:05.861366Z", + "iopub.status.busy": "2026-07-10T19:57:05.860884Z", + "iopub.status.idle": "2026-07-10T19:57:05.946411Z", + "shell.execute_reply": "2026-07-10T19:57:05.945057Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stdout: Done. created: 0, updated: 0, unchanged: 8, skipped: 0, site_needs_review: 1, window_needs_review: 1\n", + "\n" + ] + } + ], + "source": [ + "import io\n", + "\n", + "from django.core.management import call_command\n", + "\n", + "fixture_path = repo_root_path / 'docs' / 'notebooks' / 'pre_executed' / 'fixtures' / 'campaign_sample.csv'\n", + "assert fixture_path.exists(), f'Fixture not found at {fixture_path}'\n", + "\n", + "stdout_buf = io.StringIO()\n", + "stderr_buf = io.StringIO()\n", + "\n", + "call_command(\n", + " 'import_campaign_csv',\n", + " '--campaign',\n", + " '3I/ATLAS (demo)',\n", + " str(fixture_path),\n", + " stdout=stdout_buf,\n", + " stderr=stderr_buf,\n", + ")\n", + "\n", + "print('stdout:', stdout_buf.getvalue())\n", + "if stderr_buf.getvalue():\n", + " print('stderr (skipped rows):', stderr_buf.getvalue())" + ] + }, + { + "cell_type": "markdown", + "id": "c37e3856", + "metadata": {}, + "source": [ + "## Inspect the imported CampaignRun rows\n", + "\n", + "Confirm the summary above matches: 8 rows in the fixture, all with a resolvable\n", + "`Telescope / Instrument`. `Obs. Date` never skips a row either (D-13) -- every row,\n", + "including the range and TBD rows added below, resolves to a window or a flagged\n", + "TBD `CampaignRun`. Against a fresh, empty dev DB this reports all 8 as `created`;\n", + "against this notebook's persistent dev DB (already populated by a prior run) it\n", + "instead reports `unchanged: 8`, with exactly one `site_needs_review` (the\n", + "blank-Site-Code row) either way. No real contact information appears below --\n", + "only the synthetic placeholders from the fixture." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "16d14cf8", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-10T19:57:05.950443Z", + "iopub.status.busy": "2026-07-10T19:57:05.950055Z", + "iopub.status.idle": "2026-07-10T19:57:05.995908Z", + "shell.execute_reply": "2026-07-10T19:57:05.994574Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total CampaignRun rows for '3I/ATLAS (demo)': 12\n", + "\n", + "telescope_instrument run_status site_needs_review target \n", + "-------------------------------------------------------------------------------------------\n", + "Demo Telescope/DemoCam requested False (none) \n", + "Demo Telescope/DemoSpec requested False (none) \n", + "VLT/X-shooter planned False 3I/ATLAS (demo target) \n", + "FTN/MuSCAT3 observed False 3I/ATLAS (demo target) \n", + "ESO VLT FORS2 observed False 3I/ATLAS (demo target) \n", + "DCT requested True (none) \n", + "DCT requested False (none) \n", + "Apache Point Observatory/ARCTIC cancelled False 3I/ATLAS (demo target) \n", + "Apache Point Observatory/KOSMOS observed False 3I/ATLAS (demo target) \n", + "Generic 1m robotic telescope planned True 3I/ATLAS (demo target) \n", + "VLT/MUSE observed False 3I/ATLAS (demo target) \n", + "FTN/FLOYDS planned False 3I/ATLAS (demo target) \n" + ] + } + ], + "source": [ + "from solsys_code.models import CampaignRun\n", + "\n", + "runs = CampaignRun.objects.filter(campaign=campaign).order_by('window_start')\n", + "print(f'Total CampaignRun rows for {campaign.name!r}: {runs.count()}')\n", + "print()\n", + "header = f'{\"telescope_instrument\":<34} {\"run_status\":<10} {\"site_needs_review\":<18} {\"target\":<26}'\n", + "print(header)\n", + "print('-' * len(header))\n", + "for run in runs:\n", + " target_label = run.target.name if run.target else '(none)'\n", + " print(f'{run.telescope_instrument:<34} {run.run_status:<10} {str(run.site_needs_review):<18} {target_label:<26}')" + ] + }, + { + "cell_type": "markdown", + "id": "3a0e13cc", + "metadata": {}, + "source": [ + "## Range/TBD import demonstration (IMPORT-01, IMPORT-02)\n", + "\n", + "The two rows added to the fixture above exercise Plan 03's `parse_obs_window()`\n", + "range/TBD parsing and Plan 02's `original_obs_date_raw`/`window_needs_review`\n", + "columns end-to-end:\n", + "\n", + "- **Gia Range** (`FTN/FLOYDS`, `Obs. Date = '2025-08-01 to 2025-08-15'`) is a\n", + " full-date range (D-12) -- it imports with a resolved `window_start`/`window_end`\n", + " pair, not skipped (IMPORT-01).\n", + "- **Ike Pending** (`VLT/X-shooter`, `Obs. Date = 'TBD pending Cycle 2'`) is\n", + " genuinely unparseable -- `parse_obs_window()`'s never-raise TBD contract (D-13)\n", + " imports it as a `CampaignRun` with `window_start`/`window_end = None`,\n", + " `window_needs_review = True`, and the verbatim raw text preserved in\n", + " `original_obs_date_raw`, rather than dropping the row (IMPORT-02).\n", + "\n", + "The import-summary line above also reports these via the `window_needs_review`\n", + "counter (now `1`, for the TBD row only -- the range row resolves to a window and\n", + "does not count)." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "f9ce5983", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-10T19:57:06.000230Z", + "iopub.status.busy": "2026-07-10T19:57:05.999691Z", + "iopub.status.idle": "2026-07-10T19:57:06.039653Z", + "shell.execute_reply": "2026-07-10T19:57:06.037914Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Range row (IMPORT-01):\n", + " telescope_instrument='FTN/FLOYDS'\n", + " window_start=datetime.date(2025, 8, 1), window_end=datetime.date(2025, 8, 15)\n", + " window_needs_review=False\n", + "\n", + "TBD row (IMPORT-02):\n", + " telescope_instrument='VLT/X-shooter'\n", + " window_start=None, window_end=None\n", + " window_needs_review=True\n", + " original_obs_date_raw='TBD pending Cycle 2'\n" + ] + } + ], + "source": [ + "range_run = CampaignRun.objects.get(campaign=campaign, telescope_instrument='FTN/FLOYDS')\n", + "tbd_run = CampaignRun.objects.get(campaign=campaign, telescope_instrument='VLT/X-shooter')\n", + "\n", + "print('Range row (IMPORT-01):')\n", + "print(f' telescope_instrument={range_run.telescope_instrument!r}')\n", + "print(f' window_start={range_run.window_start!r}, window_end={range_run.window_end!r}')\n", + "print(f' window_needs_review={range_run.window_needs_review!r}')\n", + "print()\n", + "print('TBD row (IMPORT-02):')\n", + "print(f' telescope_instrument={tbd_run.telescope_instrument!r}')\n", + "print(f' window_start={tbd_run.window_start!r}, window_end={tbd_run.window_end!r}')\n", + "print(f' window_needs_review={tbd_run.window_needs_review!r}')\n", + "print(f' original_obs_date_raw={tbd_run.original_obs_date_raw!r}')" + ] + }, + { + "cell_type": "markdown", + "id": "c32cae1e", + "metadata": {}, + "source": [ + "## Approval lifecycle: pending_review -> approved / rejected (D-03)\n", + "\n", + "Bootstrap-imported rows always land with `approval_status=APPROVED` — they are\n", + "vetted historical data being backfilled, not fresh submissions awaiting review\n", + "(D-03). The full `pending_review` -> `approved`/`rejected` lifecycle only\n", + "becomes operationally relevant once Phase 16's community submission form\n", + "exists. It is demonstrated here directly on two synthetic `CampaignRun` rows,\n", + "created outside the CSV import, so the transition has automated/notebook\n", + "coverage somewhere in this milestone.\n", + "\n", + "**Idempotency note (window-schema migration):** Phase 19 added a partial `UniqueConstraint` on `(campaign, telescope_instrument, contact_person)` for TBD (`window_start IS NULL`) rows. The two demo rows below are therefore found-or-created via `update_or_create` (matching the Observatory-seeding cell above) rather than unconditional `.create()`, so this notebook stays safely re-runnable against a dev DB that already has a prior execution's rows." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "9698c862", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-10T19:57:06.043690Z", + "iopub.status.busy": "2026-07-10T19:57:06.043253Z", + "iopub.status.idle": "2026-07-10T19:57:06.149397Z", + "shell.execute_reply": "2026-07-10T19:57:06.148094Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created (approval demo #1): approval_status=CampaignRun.ApprovalStatus.PENDING_REVIEW\n", + "After staff approval: approval_status='approved'\n", + "\n", + "Created (approval demo #2): approval_status=CampaignRun.ApprovalStatus.PENDING_REVIEW\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "After staff rejection: approval_status='rejected'\n" + ] + } + ], + "source": [ + "pending_run, _ = CampaignRun.objects.update_or_create(\n", + " campaign=campaign,\n", + " telescope_instrument='Demo Telescope/DemoCam',\n", + " contact_person='Grace Lifecycle',\n", + " defaults={\n", + " 'contact_email': 'grace.lifecycle@example.com',\n", + " 'approval_status': CampaignRun.ApprovalStatus.PENDING_REVIEW,\n", + " },\n", + ")\n", + "print(f'Created (approval demo #1): approval_status={pending_run.approval_status!r}')\n", + "\n", + "pending_run.approval_status = CampaignRun.ApprovalStatus.APPROVED\n", + "pending_run.save(update_fields=['approval_status'])\n", + "pending_run.refresh_from_db()\n", + "print(f'After staff approval: approval_status={pending_run.approval_status!r}')\n", + "\n", + "print()\n", + "\n", + "rejected_run, _ = CampaignRun.objects.update_or_create(\n", + " campaign=campaign,\n", + " telescope_instrument='Demo Telescope/DemoSpec',\n", + " contact_person='Hal Lifecycle',\n", + " defaults={\n", + " 'contact_email': 'hal.lifecycle@example.com',\n", + " 'approval_status': CampaignRun.ApprovalStatus.PENDING_REVIEW,\n", + " },\n", + ")\n", + "print(f'Created (approval demo #2): approval_status={rejected_run.approval_status!r}')\n", + "\n", + "rejected_run.approval_status = CampaignRun.ApprovalStatus.REJECTED\n", + "rejected_run.save(update_fields=['approval_status'])\n", + "rejected_run.refresh_from_db()\n", + "print(f'After staff rejection: approval_status={rejected_run.approval_status!r}')" + ] + }, + { + "cell_type": "markdown", + "id": "65c003ef", + "metadata": {}, + "source": [ + "## Idempotency check\n", + "\n", + "Running the command a second time with the same fixture should produce zero\n", + "new rows and zero updates — the summary should report `created: 0, updated: 0,\n", + "unchanged: 8`. The two synthetic approval-lifecycle rows above are untouched\n", + "(different `telescope_instrument` values, not present in the fixture)." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "29a378f7", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-10T19:57:06.153122Z", + "iopub.status.busy": "2026-07-10T19:57:06.152772Z", + "iopub.status.idle": "2026-07-10T19:57:06.227380Z", + "shell.execute_reply": "2026-07-10T19:57:06.225749Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Second run stdout: Done. created: 0, updated: 0, unchanged: 8, skipped: 0, site_needs_review: 1, window_needs_review: 1\n", + "\n", + "Total CampaignRun rows for '3I/ATLAS (demo)' after re-run: 12\n" + ] + } + ], + "source": [ + "stdout_buf2 = io.StringIO()\n", + "stderr_buf2 = io.StringIO()\n", + "\n", + "call_command(\n", + " 'import_campaign_csv',\n", + " '--campaign',\n", + " '3I/ATLAS (demo)',\n", + " str(fixture_path),\n", + " stdout=stdout_buf2,\n", + " stderr=stderr_buf2,\n", + ")\n", + "\n", + "print('Second run stdout:', stdout_buf2.getvalue())\n", + "print(\n", + " f'Total CampaignRun rows for {campaign.name!r} after re-run: '\n", + " f'{CampaignRun.objects.filter(campaign=campaign).count()}'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "90d785d4", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "This notebook demonstrates the Phase 14 Plan 03 deliverable (CAMP-05) and\n", + "exercises CAMP-04's import command end-to-end against a synthetic fixture:\n", + "\n", + "| Requirement | Description | Demonstrated by |\n", + "|-------------|-------------|------------------|\n", + "| CAMP-04 | Command reports created/updated/unchanged/skipped/site_needs_review; idempotent re-run | Import cell summary and re-run cell summary both report `unchanged: 8` against this notebook's persistent dev DB (already populated by a prior run); a fresh, empty dev DB would instead report `created: 8` on the first run |\n", + "| CAMP-02 / D-07 | Optional `target` FK auto-resolves for a single-Target campaign | Inspection table's `target` column shows the same demo Target for every row |\n", + "| D-08 / D-09 | Site resolution never skips a row; unresolved site is flagged, not fabricated | Inspection table shows `site_needs_review=True` for the blank-Site-Code row only |\n", + "| D-03 / CAMP-03 | `pending_review` -> `approved`/`rejected` approval lifecycle | Approval-lifecycle cell before/after prints |\n", + "| CAMP-05 | No real PII in git history; no live network call | Every contact/email value above is a synthetic placeholder; Site Codes are limited to the three Observatory rows seeded locally in this notebook |\n", + "| IMPORT-01 | A date-range `Obs. Date` imports as a resolved multi-night window | Range/TBD demonstration cell shows `window_start`/`window_end` set for the range row |\n", + "| IMPORT-02 | An unparseable `Obs. Date` never skips a row -- it imports flagged `window_needs_review=True` with the raw text preserved | Range/TBD demonstration cell shows `window_needs_review=True` and `original_obs_date_raw` for the TBD row |\n", + "\n", + "This notebook is **pre-executed** and intentionally excluded from automated doc\n", + "builds (not referenced in `docs/notebooks.rst`) because it depends on\n", + "`Observatory`/`TargetList`/`CampaignRun` records and writes to the local dev DB." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "fomo312_venv (3.12.3.final.0)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/notebooks/pre_executed/load_telescope_runs_demo.ipynb b/docs/notebooks/pre_executed/load_telescope_runs_demo.ipynb new file mode 100644 index 00000000..3a9e8789 --- /dev/null +++ b/docs/notebooks/pre_executed/load_telescope_runs_demo.ipynb @@ -0,0 +1,917 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Telescope Runs — Stage 2 Demo (classical run ingest)\n", + "\n", + "This notebook demonstrates `solsys_code/management/commands/load_telescope_runs.py`\n", + "(issue #37 Stage 2), the classical-schedule ingest command that parses run lines\n", + "and idempotently creates one `CalendarEvent` per observing night.\n", + "\n", + "It demonstrates:\n", + "\n", + "- Seeding `Observatory` records for the 4 supported sites (idempotent `update_or_create`)\n", + "- Writing a small sample schedule file to a temporary path\n", + "- Invoking `load_telescope_runs` via `call_command`\n", + "- Inspecting the resulting `CalendarEvent` rows (title, start/end times, description)\n", + "- Re-running the command to confirm idempotency (no duplicate events)\n", + "\n", + "This notebook lives in `pre_executed/` because it is **DB-dependent** (it seeds\n", + "`Observatory` records and creates `CalendarEvent` rows) and is therefore **NOT\n", + "run during Sphinx/CI/ReadTheDocs builds**, per `docs/notebooks/README.md`." + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## Django setup\n", + "\n", + "Standard boilerplate to make `src.fomo.settings` importable from this notebook's\n", + "location (`docs/notebooks/pre_executed/` — three levels under the repo root, so\n", + "`parents[2]` gives the repo root) and to allow synchronous ORM calls inside\n", + "Jupyter's async event loop." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "c3d4e5f6", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-16T21:29:04.762057Z", + "iopub.status.busy": "2026-07-16T21:29:04.761519Z", + "iopub.status.idle": "2026-07-16T21:29:05.303914Z", + "shell.execute_reply": "2026-07-16T21:29:05.302048Z" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "import django\n", + "\n", + "# Ensure the repo root is on sys.path so `src.fomo.settings` is importable\n", + "# when this notebook is executed from docs/notebooks/pre_executed/.\n", + "# NOTE: parents[2] is correct only when the Jupyter kernel CWD is\n", + "# docs/notebooks/pre_executed/. Start Jupyter from that directory, or\n", + "# adjust the index if you launch from the repo root.\n", + "repo_root_path = Path.cwd().resolve().parents[2]\n", + "assert (\n", + " repo_root_path / 'manage.py'\n", + ").exists(), (\n", + " f'Repo root not found at {repo_root_path}. Run Jupyter from docs/notebooks/pre_executed/ or adjust parents[] index.'\n", + ")\n", + "repo_root = str(repo_root_path)\n", + "if repo_root not in sys.path:\n", + " sys.path.insert(0, repo_root)\n", + "\n", + "os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'src.fomo.settings')\n", + "\n", + "# Jupyter's ipykernel runs inside an asyncio event loop, but Django's ORM is\n", + "# sync-only by default and refuses to run there; this opts back in.\n", + "os.environ.setdefault('DJANGO_ALLOW_ASYNC_UNSAFE', 'true')\n", + "\n", + "django.setup()" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Seed Observatory records\n", + "\n", + "`load_telescope_runs` resolves telescope names to `Observatory` rows via MPC obscode\n", + "(through `telescope_runs.get_site()`). The 4 sites below match the values used in\n", + "`solsys_code/tests/test_telescope_runs.py`'s `setUpTestData`. Using\n", + "`update_or_create` makes this cell idempotent — safe to re-run against any dev DB." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "e5f6a7b8", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-16T21:29:05.307431Z", + "iopub.status.busy": "2026-07-16T21:29:05.306914Z", + "iopub.status.idle": "2026-07-16T21:29:05.405513Z", + "shell.execute_reply": "2026-07-16T21:29:05.404425Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " obscode='268' updated/unchanged short_name='Magellan-Clay'\n", + " obscode='269' updated/unchanged short_name='Magellan-Baade'\n", + " obscode='809' updated/unchanged short_name='NTT'\n", + " obscode='E10' updated/unchanged short_name='FTS'\n" + ] + } + ], + "source": [ + "from solsys_code.solsys_code_observatory.models import Observatory\n", + "\n", + "SITE_DATA = {\n", + " '268': dict(\n", + " name='Magellan Clay Telescope',\n", + " short_name='Magellan-Clay',\n", + " lat=-29.0146,\n", + " lon=-70.6926,\n", + " altitude=2402,\n", + " timezone='America/Santiago',\n", + " ),\n", + " '269': dict(\n", + " name='Magellan Baade Telescope',\n", + " short_name='Magellan-Baade',\n", + " lat=-29.0146,\n", + " lon=-70.6926,\n", + " altitude=2402,\n", + " timezone='America/Santiago',\n", + " ),\n", + " '809': dict(\n", + " name='ESO, La Silla',\n", + " short_name='NTT',\n", + " lat=-29.2567,\n", + " lon=-70.7300,\n", + " altitude=2347,\n", + " timezone='America/Santiago',\n", + " ),\n", + " 'E10': dict(\n", + " name='Siding Spring Observatory',\n", + " short_name='FTS',\n", + " lat=-31.2734,\n", + " lon=149.0612,\n", + " altitude=1149,\n", + " timezone='Australia/Sydney',\n", + " ),\n", + "}\n", + "\n", + "for obscode, fields in SITE_DATA.items():\n", + " obs, created = Observatory.objects.update_or_create(obscode=obscode, defaults=fields)\n", + " action = 'created' if created else 'updated/unchanged'\n", + " print(f' obscode={obscode!r:>4} {action:>18} short_name={obs.short_name!r}')" + ] + }, + { + "cell_type": "markdown", + "id": "56b32ef2", + "metadata": {}, + "source": [ + "## Night convention: ESO noon-to-noon vs Las Campanas both-inclusive\n", + "\n", + "Different sites count the nights of a date range differently, and\n", + "`_iter_run_nights` applies the right convention per site:\n", + "\n", + "- **Las Campanas (Magellan)** — Start and End are *both* inclusive\n", + " observing nights, so a range yields `E - S + 1` nights (see\n", + " `docs/design/telescope_runs_calendar.rst` \"Night convention\").\n", + "- **ESO sites (`ESO_NOON_TO_NOON_SITES`, e.g. NTT / La Silla)** — the range\n", + " is transcribed verbatim from ESO's *Tatoo* tool, whose displayed **End date\n", + " is the noon-to-noon closing boundary of the last night**, not itself an\n", + " observing night. So the last observing night is `End - 1` and the range\n", + " yields `E - S` nights.\n", + "\n", + "For example, ESO's Tatoo reports *4.0 nights* for `NTT ... 9-13 July`; the\n", + "cell below confirms `_iter_run_nights` produces exactly those 4 nights\n", + "(9–12 July), while the same-length Magellan range stays both-inclusive." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "ae40a843", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-16T21:29:05.408845Z", + "iopub.status.busy": "2026-07-16T21:29:05.408538Z", + "iopub.status.idle": "2026-07-16T21:29:05.443476Z", + "shell.execute_reply": "2026-07-16T21:29:05.442326Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ESO noon-to-noon sites: ['NTT']\n", + "\n", + "'NTT EFOSC2 allocation 9-13 July'\n", + " telescope=NTT day1=9 day2=13 -> 4 nights\n", + " convention: ESO Tatoo noon-to-noon (drops End boundary -> E - S nights)\n", + " nights: ['2026-07-09', '2026-07-10', '2026-07-11', '2026-07-12']\n", + "\n", + "'Magellan-Baade IMACS 17-18 July'\n", + " telescope=Magellan-Baade day1=17 day2=18 -> 2 nights\n", + " convention: Las Campanas both-inclusive (E - S + 1 nights)\n", + " nights: ['2026-07-17', '2026-07-18']\n", + "\n" + ] + } + ], + "source": [ + "from solsys_code.management.commands.load_telescope_runs import _iter_run_nights\n", + "from solsys_code.telescope_runs import ESO_NOON_TO_NOON_SITES, parse_run_line\n", + "\n", + "print('ESO noon-to-noon sites:', sorted(ESO_NOON_TO_NOON_SITES))\n", + "print()\n", + "\n", + "for line in ['NTT EFOSC2 allocation 9-13 July', 'Magellan-Baade IMACS 17-18 July']:\n", + " parsed = parse_run_line(line)\n", + " nights = _iter_run_nights(parsed)\n", + " convention = (\n", + " 'ESO Tatoo noon-to-noon (drops End boundary -> E - S nights)'\n", + " if parsed.telescope in ESO_NOON_TO_NOON_SITES\n", + " else 'Las Campanas both-inclusive (E - S + 1 nights)'\n", + " )\n", + " print(f'{line!r}')\n", + " print(f' telescope={parsed.telescope} day1={parsed.day1} day2={parsed.day2} -> {len(nights)} nights')\n", + " print(f' convention: {convention}')\n", + " print(f' nights: {[d.isoformat() for d in nights]}')\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "f6a7b8c9", + "metadata": {}, + "source": [ + "## Write a sample schedule file\n", + "\n", + "A real schedule file is a plain-text file with one run line per non-blank line.\n", + "Here we write a small sample to a temporary file. The format accepted by\n", + "`parse_run_line` is flexible: the month name may appear before or after the\n", + "day range, and instrument names may be hyphenated." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a7b8c9d0", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-16T21:29:05.446636Z", + "iopub.status.busy": "2026-07-16T21:29:05.446343Z", + "iopub.status.idle": "2026-07-16T21:29:05.470133Z", + "shell.execute_reply": "2026-07-16T21:29:05.468910Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Schedule file written to: /tmp/tmpcn50w9m7.txt\n", + "\n", + "Contents:\n", + "NTT EFOSC2 allocation 9-13 July\n", + "FTS MUSCAT4 allocation 10-12 July\n", + "Magellan IMACS 14-16 July (proposed)\n", + "\n" + ] + } + ], + "source": [ + "import tempfile\n", + "\n", + "SAMPLE_SCHEDULE = \"\"\"\\\n", + "NTT EFOSC2 allocation 9-13 July\n", + "FTS MUSCAT4 allocation 10-12 July\n", + "Magellan IMACS 14-16 July (proposed)\n", + "\"\"\"\n", + "\n", + "with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as tmp:\n", + " tmp.write(SAMPLE_SCHEDULE)\n", + " schedule_path = tmp.name\n", + "\n", + "print('Schedule file written to:', schedule_path)\n", + "print()\n", + "print('Contents:')\n", + "print(SAMPLE_SCHEDULE)" + ] + }, + { + "cell_type": "markdown", + "id": "b8c9d0e1", + "metadata": {}, + "source": [ + "## Invoke the load_telescope_runs command\n", + "\n", + "`call_command` is the Django-recommended way to invoke management commands\n", + "programmatically. The command will:\n", + "\n", + "1. Parse each run line with `parse_run_line`\n", + "2. Resolve the telescope name to its `Observatory` via `get_site`\n", + "3. Expand the date range to one evening per night\n", + "4. For each night: compute `sun_event` sunset/sunrise and the -15° dark window\n", + "5. Create or update a `CalendarEvent` keyed on `(telescope, instrument, start_time)`\n", + "\n", + "The end-of-run summary reports `created`, `updated`, `unchanged`, and `skipped` counts." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "c9d0e1f2", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-16T21:29:05.473085Z", + "iopub.status.busy": "2026-07-16T21:29:05.472777Z", + "iopub.status.idle": "2026-07-16T21:29:13.610088Z", + "shell.execute_reply": "2026-07-16T21:29:13.608474Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stdout: Done. lines processed: 3, created: 0, updated: 7, unchanged: 0, skipped: 1\n", + "\n", + "stderr (skipped lines): Line 3: Ambiguous telescope 'Magellan': matches multiple SITES keys ['Magellan-Clay', 'Magellan-Baade']; use a more specific telescope name (e.g. \"Magellan-Clay\" or \"Magellan-Baade\"). (line text: 'Magellan IMACS 14-16 July (proposed)')\n", + "\n" + ] + } + ], + "source": [ + "import io\n", + "\n", + "from django.core.management import call_command\n", + "\n", + "stdout_buf = io.StringIO()\n", + "stderr_buf = io.StringIO()\n", + "\n", + "call_command('load_telescope_runs', schedule_path, stdout=stdout_buf, stderr=stderr_buf)\n", + "\n", + "print('stdout:', stdout_buf.getvalue())\n", + "if stderr_buf.getvalue():\n", + " print('stderr (skipped lines):', stderr_buf.getvalue())" + ] + }, + { + "cell_type": "markdown", + "id": "d0e1f2a3", + "metadata": {}, + "source": [ + "## Inspect the created CalendarEvent rows\n", + "\n", + "Each observing night in the schedule produces one `CalendarEvent`. The `description`\n", + "field (D-06) contains the -15° dark-window UTC times, the run status, and the\n", + "original source line for traceability." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e1f2a3b4", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-16T21:29:13.614615Z", + "iopub.status.busy": "2026-07-16T21:29:13.614164Z", + "iopub.status.idle": "2026-07-16T21:29:13.651579Z", + "shell.execute_reply": "2026-07-16T21:29:13.650132Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total CalendarEvent rows: 26\n", + "\n", + "Title : 3I/ATLAS (demo): DCT\n", + "start_time : 2025-07-05T02:50:31+00:00\n", + "end_time : 2025-07-05T12:10:16+00:00\n", + "Description:\n", + "\n", + "Title : 3I/ATLAS (demo): DCT\n", + "start_time : 2025-07-06T02:50:20+00:00\n", + "end_time : 2025-07-06T12:10:48+00:00\n", + "Description:\n", + "\n", + "Title : Crash Test Campaign: Sinistro\n", + "start_time : 2026-05-11T15:56:04+00:00\n", + "end_time : 2026-05-12T05:10:31+00:00\n", + "Description:\n", + "\n", + "Title : Test Campaign: 0.5m + CMOS\n", + "start_time : 2026-05-25T18:49:30+00:00\n", + "end_time : 2026-05-26T03:36:46+00:00\n", + "Description:\n", + "\n", + "Title : Test Campaign: NOT/ALFOSC\n", + "start_time : 2026-05-25T20:08:45+00:00\n", + "end_time : 2026-05-26T06:08:06+00:00\n", + "Description:\n", + "\n", + "Title : FTS MuSCAT\n", + "start_time : 2026-07-08T08:45:00+00:00\n", + "end_time : 2026-07-08T15:35:00+00:00\n", + "Description:\n", + " Queue observation — LCO2026A-003\n", + " Fixed field, 08:45–15:35 UTC\n", + "\n", + "Title : FTS MuSCAT\n", + "start_time : 2026-07-09T08:45:00+00:00\n", + "end_time : 2026-07-09T15:35:00+00:00\n", + "Description:\n", + " Queue observation — LCO2026A-003\n", + " Fixed field, 08:45–15:35 UTC\n", + "\n", + "Title : NTT EFOSC2\n", + "start_time : 2026-07-09T22:06:35+00:00\n", + "end_time : 2026-07-10T11:29:46+00:00\n", + "Description:\n", + " Dark window (-15 deg, UTC): 2026-07-09T23:09:10+00:00 to 2026-07-10T10:27:15+00:00\n", + " Status: allocation\n", + " Source line: NTT EFOSC2 allocation 9-13 July\n", + "\n", + "Title : FTS MUSCAT4\n", + "start_time : 2026-07-10T07:21:08+00:00\n", + "end_time : 2026-07-10T20:56:59+00:00\n", + "Description:\n", + " Dark window (-15 deg, UTC): 2026-07-10T08:27:16+00:00 to 2026-07-10T19:50:55+00:00\n", + " Status: allocation\n", + " Source line: FTS MUSCAT4 allocation 10-12 July\n", + "\n", + "Title : FTS MuSCAT\n", + "start_time : 2026-07-10T08:45:00+00:00\n", + "end_time : 2026-07-10T15:35:00+00:00\n", + "Description:\n", + " Queue observation — LCO2026A-003\n", + " Fixed field, 08:45–15:35 UTC\n", + "\n", + "Title : NTT EFOSC2\n", + "start_time : 2026-07-10T22:07:04+00:00\n", + "end_time : 2026-07-11T11:29:34+00:00\n", + "Description:\n", + " Dark window (-15 deg, UTC): 2026-07-10T23:09:35+00:00 to 2026-07-11T10:27:07+00:00\n", + " Status: allocation\n", + " Source line: NTT EFOSC2 allocation 9-13 July\n", + "\n", + "Title : FTS MUSCAT4\n", + "start_time : 2026-07-11T07:21:39+00:00\n", + "end_time : 2026-07-11T20:56:44+00:00\n", + "Description:\n", + " Dark window (-15 deg, UTC): 2026-07-11T08:27:42+00:00 to 2026-07-11T19:50:45+00:00\n", + " Status: allocation\n", + " Source line: FTS MUSCAT4 allocation 10-12 July\n", + "\n", + "Title : FTS MuSCAT\n", + "start_time : 2026-07-11T08:45:00+00:00\n", + "end_time : 2026-07-11T15:35:00+00:00\n", + "Description:\n", + " Queue observation — LCO2026A-003\n", + " Fixed field, 08:45–15:35 UTC\n", + "\n", + "Title : NTT EFOSC2\n", + "start_time : 2026-07-11T22:07:32+00:00\n", + "end_time : 2026-07-12T11:29:21+00:00\n", + "Description:\n", + " Dark window (-15 deg, UTC): 2026-07-11T23:10:00+00:00 to 2026-07-12T10:26:57+00:00\n", + " Status: allocation\n", + " Source line: NTT EFOSC2 allocation 9-13 July\n", + "\n", + "Title : FTS MUSCAT4\n", + "start_time : 2026-07-12T07:22:10+00:00\n", + "end_time : 2026-07-12T20:56:28+00:00\n", + "Description:\n", + " Dark window (-15 deg, UTC): 2026-07-12T08:28:09+00:00 to 2026-07-12T19:50:33+00:00\n", + " Status: allocation\n", + " Source line: FTS MUSCAT4 allocation 10-12 July\n", + "\n", + "Title : FTS MuSCAT\n", + "start_time : 2026-07-12T08:45:00+00:00\n", + "end_time : 2026-07-12T15:35:00+00:00\n", + "Description:\n", + " Queue observation — LCO2026A-003\n", + " Fixed field, 08:45–15:35 UTC\n", + "\n", + "Title : NTT EFOSC2\n", + "start_time : 2026-07-12T22:08:02+00:00\n", + "end_time : 2026-07-13T11:29:06+00:00\n", + "Description:\n", + " Dark window (-15 deg, UTC): 2026-07-12T23:10:25+00:00 to 2026-07-13T10:26:46+00:00\n", + " Status: allocation\n", + " Source line: NTT EFOSC2 allocation 9-13 July\n", + "\n", + "Title : FTS MuSCAT\n", + "start_time : 2026-07-13T08:45:00+00:00\n", + "end_time : 2026-07-13T15:35:00+00:00\n", + "Description:\n", + " Queue observation — LCO2026A-003\n", + " Fixed field, 08:45–15:35 UTC\n", + "\n", + "Title : FTS MuSCAT\n", + "start_time : 2026-07-14T08:45:00+00:00\n", + "end_time : 2026-07-14T15:35:00+00:00\n", + "Description:\n", + " Queue observation — LCO2026A-003\n", + " Fixed field, 08:45–15:35 UTC\n", + "\n", + "Title : Magellan-Baade IMACS\n", + "start_time : 2026-07-17T22:10:58+00:00\n", + "end_time : 2026-07-18T11:26:49+00:00\n", + "Description:\n", + " Dark window (-15 deg, UTC): 2026-07-17T23:12:46+00:00 to 2026-07-18T10:25:06+00:00\n", + " Status: allocation\n", + " Source line: Magellan-Baade IMACS 17-18 July\n", + "\n", + "Title : Magellan-Baade IMACS\n", + "start_time : 2026-07-18T22:11:29+00:00\n", + "end_time : 2026-07-19T11:26:26+00:00\n", + "Description:\n", + " Dark window (-15 deg, UTC): 2026-07-18T23:13:13+00:00 to 2026-07-19T10:24:48+00:00\n", + " Status: allocation\n", + " Source line: Magellan-Baade IMACS 17-18 July\n", + "\n", + "Title : Magellan-Clay Lightspeed\n", + "start_time : 2026-07-18T22:11:29+00:00\n", + "end_time : 2026-07-19T06:26:00+00:00\n", + "Description:\n", + " Dark window (-15 deg, UTC): 2026-07-18T23:13:13+00:00 to 2026-07-19T10:24:48+00:00\n", + " Status: allocation\n", + " Source line: Magellan-Clay Lightspeed 18-20 July BoN-0626\n", + "\n", + "Title : Magellan-Clay Lightspeed\n", + "start_time : 2026-07-19T22:12:01+00:00\n", + "end_time : 2026-07-20T06:26:00+00:00\n", + "Description:\n", + " Dark window (-15 deg, UTC): 2026-07-19T23:13:39+00:00 to 2026-07-20T10:24:28+00:00\n", + " Status: allocation\n", + " Source line: Magellan-Clay Lightspeed 18-20 July BoN-0626\n", + "\n", + "Title : Magellan-Clay Lightspeed\n", + "start_time : 2026-07-20T22:12:32+00:00\n", + "end_time : 2026-07-21T06:26:00+00:00\n", + "Description:\n", + " Dark window (-15 deg, UTC): 2026-07-20T23:14:06+00:00 to 2026-07-21T10:24:08+00:00\n", + " Status: allocation\n", + " Source line: Magellan-Clay Lightspeed 18-20 July BoN-0626\n", + "\n", + "Title : [CANCELLED] NTT EFOSC2\n", + "start_time : 2026-07-21T22:12:41+00:00\n", + "end_time : 2026-07-22T11:25:50+00:00\n", + "Description:\n", + " Dark window (-15 deg, UTC): 2026-07-21T23:14:24+00:00 to 2026-07-22T10:24:12+00:00\n", + " Status: cancelled\n", + " Source line: NTT EFOSC2 21-22 July (cancelled)\n", + "\n", + "Title : Test Campaign: CR-01 UAT Test 1m\n", + "start_time : 2026-08-02T02:28:38+00:00\n", + "end_time : 2026-08-02T12:30:34+00:00\n", + "Description:\n", + " UAT fixture for CR-01 blank-timezone resolve_site regression test (Phase 22 test 3).\n", + "\n" + ] + } + ], + "source": [ + "from tom_calendar.models import CalendarEvent\n", + "\n", + "events = CalendarEvent.objects.order_by('start_time')\n", + "print(f'Total CalendarEvent rows: {events.count()}')\n", + "print()\n", + "\n", + "for ev in events:\n", + " print(f'Title : {ev.title}')\n", + " print(f'start_time : {ev.start_time.isoformat()}')\n", + " print(f'end_time : {ev.end_time.isoformat()}')\n", + " print('Description:')\n", + " for line in ev.description.splitlines():\n", + " print(f' {line}')\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "cc01cd01", + "metadata": {}, + "source": [ + "## Cancelled classical run: [CANCELLED] title prefix (D-01/D-02)\n", + "\n", + "A staff member marks a classical run cancelled by adding the recognized `cancelled`\n", + "status word/parenthetical to the source schedule line and re-running\n", + "`load_telescope_runs`. The resulting `CalendarEvent.title` now begins with\n", + "`[CANCELLED] `, mirroring the LCO sync's existing `[CANCELLED]`/`[EXPIRED]` title-prefix\n", + "idiom (see `sync_lco_observation_calendar.py`'s `_FAILURE_PREFIX_BY_STATUS`). No\n", + "templatetag change was needed for this: `[CANCELLED]` is already a member of\n", + "`calendar_display_extras._TERMINAL_PREFIXES`, so the event also picks up the terminal\n", + "box-shadow ring for free.\n", + "\n", + "Below we reuse the NTT/EFOSC2 telescope+instrument already loaded above (so its\n", + "`Observatory` record already exists) but on a new night, with the `cancelled`\n", + "status word appended." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "cc02ce02", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-16T21:29:13.655299Z", + "iopub.status.busy": "2026-07-16T21:29:13.654937Z", + "iopub.status.idle": "2026-07-16T21:29:14.808845Z", + "shell.execute_reply": "2026-07-16T21:29:14.807592Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done. lines processed: 1, created: 0, updated: 0, unchanged: 1, skipped: 0\n", + "\n", + "Title : [CANCELLED] NTT EFOSC2\n", + "Description:\n", + " Dark window (-15 deg, UTC): 2026-07-21T23:14:24+00:00 to 2026-07-22T10:24:12+00:00\n", + " Status: cancelled\n", + " Source line: NTT EFOSC2 21-22 July (cancelled)\n" + ] + } + ], + "source": [ + "CANCELLED_SCHEDULE = \"\"\"\\\n", + "NTT EFOSC2 21-22 July (cancelled)\n", + "\"\"\"\n", + "\n", + "with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as tmp:\n", + " tmp.write(CANCELLED_SCHEDULE)\n", + " cancelled_path = tmp.name\n", + "\n", + "stdout_buf_c = io.StringIO()\n", + "call_command('load_telescope_runs', cancelled_path, stdout=stdout_buf_c, stderr=io.StringIO())\n", + "print(stdout_buf_c.getvalue())\n", + "\n", + "cancelled_event = CalendarEvent.objects.get(\n", + " telescope='NTT', instrument='EFOSC2', start_time__date__day=21, start_time__date__month=7\n", + ")\n", + "print(f'Title : {cancelled_event.title}')\n", + "print('Description:')\n", + "for line in cancelled_event.description.splitlines():\n", + " print(f' {line}')\n", + "assert cancelled_event.title.startswith('[CANCELLED] '), 'Expected a [CANCELLED]-prefixed title'" + ] + }, + { + "cell_type": "markdown", + "id": "f2a3b4c5", + "metadata": {}, + "source": [ + "## Idempotency check\n", + "\n", + "Running the command a second time with the same file should produce zero new\n", + "events and zero updates — the summary should report `created: 0, updated: 0`.\n", + "This satisfies INGEST-03." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a3b4c5d6", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-16T21:29:14.812585Z", + "iopub.status.busy": "2026-07-16T21:29:14.812182Z", + "iopub.status.idle": "2026-07-16T21:29:22.005892Z", + "shell.execute_reply": "2026-07-16T21:29:22.004857Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Second run stdout: Done. lines processed: 3, created: 0, updated: 0, unchanged: 7, skipped: 1\n", + "\n", + "CalendarEvent count unchanged: 26\n" + ] + } + ], + "source": [ + "stdout_buf2 = io.StringIO()\n", + "stderr_buf2 = io.StringIO()\n", + "\n", + "call_command('load_telescope_runs', schedule_path, stdout=stdout_buf2, stderr=stderr_buf2)\n", + "\n", + "print('Second run stdout:', stdout_buf2.getvalue())\n", + "count_after = CalendarEvent.objects.count()\n", + "print(f'CalendarEvent count unchanged: {count_after}')" + ] + }, + { + "cell_type": "markdown", + "id": "a6b03d15", + "metadata": {}, + "source": [ + "## Cross-session drift tolerance (idempotency-key robustness)\n", + "\n", + "The event `start_time` is a *computed* sun-event time (`telescope_runs.sun_event()`),\n", + "not a stable external identifier. Between independent ingests of the same night days or\n", + "weeks apart, astropy refreshes its IERS Earth-orientation data (UT1-UTC / polar motion),\n", + "so the recomputed sunset can drift by a second or two. `load_telescope_runs` therefore\n", + "matches an existing event whose `start_time` is within a few minutes of the freshly\n", + "computed value (a proximity window, not an exact datetime), so a drifted re-ingest\n", + "**updates** the existing night instead of silently creating a near-duplicate row.\n", + "\n", + "Below we simulate that cross-session drift by shifting the recomputed sun-event times by\n", + "+2 seconds on a third ingest of the same schedule file. The row count stays constant and\n", + "the summary reports `created: 0` — no duplicates. (The computed `end_time` and the\n", + "dark-window times embedded in `description` drift by the same +2s, so full-night events\n", + "report as `updated` rather than `unchanged`; the key point is that none are re-`created`.)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "da6f63c7", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-16T21:29:22.008764Z", + "iopub.status.busy": "2026-07-16T21:29:22.008483Z", + "iopub.status.idle": "2026-07-16T21:29:29.977376Z", + "shell.execute_reply": "2026-07-16T21:29:29.976221Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Drifted (+2s) re-ingest stdout: Done. lines processed: 3, created: 0, updated: 7, unchanged: 0, skipped: 1\n", + "\n", + "CalendarEvent count before drift: 26, after: 26\n", + "No duplicates were created (created: 0); the drifted nights were matched and updated.\n" + ] + } + ], + "source": [ + "from unittest import mock\n", + "\n", + "import astropy.units as u\n", + "\n", + "from solsys_code import telescope_runs as tr\n", + "\n", + "_real_sun_event = tr.sun_event\n", + "\n", + "\n", + "def _sun_event_plus_2s(site, d, kind):\n", + " \"\"\"Real sun_event() with every crossing shifted +2s (mimics cross-session IERS drift).\"\"\"\n", + " setting, rising = _real_sun_event(site, d, kind)\n", + " return setting + 2 * u.s, rising + 2 * u.s\n", + "\n", + "\n", + "count_before_drift = CalendarEvent.objects.count()\n", + "stdout_buf3 = io.StringIO()\n", + "with mock.patch(\n", + " 'solsys_code.management.commands.load_telescope_runs.sun_event',\n", + " side_effect=_sun_event_plus_2s,\n", + "):\n", + " call_command('load_telescope_runs', schedule_path, stdout=stdout_buf3, stderr=io.StringIO())\n", + "\n", + "print('Drifted (+2s) re-ingest stdout:', stdout_buf3.getvalue())\n", + "print(f'CalendarEvent count before drift: {count_before_drift}, after: {CalendarEvent.objects.count()}')\n", + "print('No duplicates were created (created: 0); the drifted nights were matched and updated.')" + ] + }, + { + "cell_type": "markdown", + "id": "6ca580d3", + "metadata": {}, + "source": [ + "## Partial-night window tokens\n", + "\n", + "Run lines may carry a trailing `(BoN|HHMM)-(EoN|HHMM)` token to restrict the\n", + "event to a portion of the night. `load_telescope_runs` passes the parsed\n", + "`start_window` / `end_window` through `_resolve_window_time`, which converts:\n", + "\n", + "- `BoN` → computed sunset for that night\n", + "- `EoN` → computed sunrise for that night\n", + "- `HHMM` → a fixed UTC datetime (HHMM < 1200 → next-morning UTC; ≥ 1200 → same evening)\n", + "\n", + "This is used for e.g. shared-telescope runs where only the first or second half\n", + "of each night is allocated." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "e6f7c482", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-16T21:29:29.980917Z", + "iopub.status.busy": "2026-07-16T21:29:29.980609Z", + "iopub.status.idle": "2026-07-16T21:29:33.479480Z", + "shell.execute_reply": "2026-07-16T21:29:33.477313Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done. lines processed: 1, created: 3, updated: 0, unchanged: 0, skipped: 0\n", + "\n", + "Night start_time (UTC) end_time (UTC) duration (h)\n", + "------------------------------------------------------------------------\n", + "2026-07-18 2026-07-18 22:11:29 2026-07-19 06:26:00 8.24h\n", + "2026-07-19 2026-07-19 22:12:01 2026-07-20 06:26:00 8.23h\n", + "2026-07-20 2026-07-20 22:12:32 2026-07-21 06:26:00 8.22h\n" + ] + } + ], + "source": [ + "import tempfile, io\n", + "from django.core.management import call_command\n", + "from tom_calendar.models import CalendarEvent\n", + "\n", + "PARTIAL_SCHEDULE = \"\"\"\\\n", + "Magellan-Clay Lightspeed 18-20 July BoN-0626\n", + "\"\"\"\n", + "\n", + "with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as tmp:\n", + " tmp.write(PARTIAL_SCHEDULE)\n", + " partial_path = tmp.name\n", + "\n", + "# Clear any existing Clay events so counts are clean for this demo\n", + "CalendarEvent.objects.filter(telescope='Magellan-Clay').delete()\n", + "\n", + "stdout_buf = io.StringIO()\n", + "call_command('load_telescope_runs', partial_path, stdout=stdout_buf, stderr=io.StringIO())\n", + "print(stdout_buf.getvalue())\n", + "\n", + "# Inspect: end_time should be 06:26 UTC on d+1 (not computed sunrise ~11:26 UTC)\n", + "print(f'{\"Night\":10} {\"start_time (UTC)\":22} {\"end_time (UTC)\":22} {\"duration (h)\":>12}')\n", + "print('-' * 72)\n", + "for ev in CalendarEvent.objects.filter(telescope='Magellan-Clay').order_by('start_time'):\n", + " dur = (ev.end_time - ev.start_time).total_seconds() / 3600\n", + " print(\n", + " f'{str(ev.start_time.date()):10} {ev.start_time.strftime(\"%Y-%m-%d %H:%M:%S\"):22}'\n", + " f' {ev.end_time.strftime(\"%Y-%m-%d %H:%M:%S\"):22} {dur:>11.2f}h'\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "b4c5d6e7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "This notebook demonstrates the Phase 03 `load_telescope_runs` command satisfying\n", + "requirements INGEST-01, INGEST-02, and INGEST-03:\n", + "\n", + "| Requirement | Description | Demonstrated by |\n", + "|-------------|-------------|------------------|\n", + "| INGEST-01 | One `CalendarEvent` per observing night, `start_time=sunset`, `end_time=sunrise` | 4 NTT (ESO noon-to-noon: 9–12 July) + 3 FTS events created; Magellan line skipped (ambiguous telescope name) |\n", + "| INGEST-02 | `title`, `description` (dark window, status, source line), `telescope`, `instrument` populated | `ev.description` printed above shows all three D-06 lines |\n", + "| INGEST-03 | Idempotent re-run leaves row count and field values unchanged | Second run reports `created: 0, updated: 0` |\n", + "| INGEST-03 (drift) | Re-ingest whose computed `start_time` drifted (IERS refresh) matches within a tolerance window instead of duplicating | +2s-drifted third run reports `created: 0`; row count unchanged |\n", + "\n", + "The command is invoked via `call_command` — equivalent to running\n", + "`./manage.py load_telescope_runs ` from the shell.\n", + "\n", + "This notebook is **pre-executed** and intentionally excluded from automated doc\n", + "builds (not referenced in `docs/notebooks.rst`) because it depends on\n", + "`Observatory` records and live `CalendarEvent` writes in the local dev DB." + ] + }, + { + "cell_type": "markdown", + "id": "33adde76", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "fomo312_venv (3.12.3.final.0)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/notebooks/pre_executed/sync_gemini_observation_calendar_demo.ipynb b/docs/notebooks/pre_executed/sync_gemini_observation_calendar_demo.ipynb new file mode 100644 index 00000000..a9bbfcd4 --- /dev/null +++ b/docs/notebooks/pre_executed/sync_gemini_observation_calendar_demo.ipynb @@ -0,0 +1,739 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# `sync_gemini_observation_calendar` Demo\n", + "\n", + "This notebook demonstrates the `sync_gemini_observation_calendar` management command,\n", + "which reads all `ObservationRecord(facility='GEM')` rows from the database and creates\n", + "or updates `CalendarEvent` rows idempotently.\n", + "\n", + "## Scenarios covered\n", + "\n", + "1. **Explicit window** — record with `windowDate`/`windowTime`/`windowDuration` present\n", + " (GEM-WINDOW-01: primary happy path).\n", + "2. **Rap: derived window** — record with no explicit window; obs code maps to a `'Rap: ...'`\n", + " settings entry → `[record.created, record.created + 24h]` (GEM-WINDOW-02).\n", + "3. **Std: derived window** — record with no explicit window; obs code maps to a `'Std: ...'`\n", + " settings entry → `[record.created + 24h, record.created + 7d]` (GEM-WINDOW-02).\n", + "4. **ON_HOLD + idempotent re-run** — record with `ready='false'` shows `[ON_HOLD]` prefix;\n", + " re-running the command produces no new events and leaves `CalendarEvent.modified` unchanged\n", + " (GEM-STATUS-01, GEM-NOCHURN-01)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "b2c3d4e5", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-27T05:09:10.724986Z", + "iopub.status.busy": "2026-06-27T05:09:10.724586Z", + "iopub.status.idle": "2026-06-27T05:09:11.443508Z", + "shell.execute_reply": "2026-06-27T05:09:11.442804Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Django setup complete\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "import django\n", + "\n", + "repo_root_path = Path.cwd().resolve().parents[2]\n", + "assert (\n", + " repo_root_path / 'manage.py'\n", + ").exists(), f'manage.py not found under {repo_root_path} — run this notebook from docs/notebooks/pre_executed/'\n", + "repo_root = str(repo_root_path)\n", + "src_root = str(repo_root_path / 'src')\n", + "for p in [repo_root, src_root]:\n", + " if p not in sys.path:\n", + " sys.path.insert(0, p)\n", + "\n", + "os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'src.fomo.settings')\n", + "os.environ.setdefault('DJANGO_ALLOW_ASYNC_UNSAFE', 'true')\n", + "django.setup()\n", + "print('Django setup complete')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c3d4e5f6", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-27T05:09:11.446070Z", + "iopub.status.busy": "2026-06-27T05:09:11.445563Z", + "iopub.status.idle": "2026-06-27T05:09:11.476131Z", + "shell.execute_reply": "2026-06-27T05:09:11.475447Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FACILITIES[\"GEM\"][\"programs\"] patched: {'GS-2026A-T-999': {'MM': 'Std: GMOS-S MOS', 'QQ': 'Rap: GMOS-S MOS'}}\n" + ] + } + ], + "source": [ + "from django.conf import settings\n", + "\n", + "# D-07: patch in a minimal FACILITIES['GEM']['programs'] block so instrument lookup\n", + "# and ToO-type detection run without real credentials.\n", + "# Program IDs follow real Gemini naming but end in '999' (D-05: fictitious, not confusable).\n", + "settings.FACILITIES.setdefault('GEM', {})\n", + "settings.FACILITIES['GEM']['programs'] = {\n", + " 'GS-2026A-T-999': {\n", + " 'MM': 'Std: GMOS-S MOS', # Standard ToO template\n", + " 'QQ': 'Rap: GMOS-S MOS', # Rapid ToO template (same instrument, different cadence)\n", + " }\n", + "}\n", + "print('FACILITIES[\"GEM\"][\"programs\"] patched:', settings.FACILITIES['GEM']['programs'])" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "d4e5f6a7", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-27T05:09:11.481986Z", + "iopub.status.busy": "2026-06-27T05:09:11.481593Z", + "iopub.status.idle": "2026-06-27T05:09:11.862501Z", + "shell.execute_reply": "2026-06-27T05:09:11.861762Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "User sync-gem-demo-user is not logged in. Cannot re-encrypt sensitive data. Clearing all encrypted fields instead.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Error clearing encrypted fields for model ESOProfile for user sync-gem-demo-user: Model instances passed to related filters must be saved.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "No Profile found for sync-gem-demo-user. Creating Profile.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Target post save hook: sync-gem-demo-target created: True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Target post save hook: sync-gem-demo-target created: False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Demo user pk=2, target pk=1\n" + ] + } + ], + "source": [ + "from django.contrib.auth import get_user_model\n", + "from tom_calendar.models import CalendarEvent\n", + "from tom_observations.models import ObservationRecord\n", + "from tom_targets.models import Target\n", + "from tom_targets.tests.factories import NonSiderealTargetFactory\n", + "\n", + "DEMO_TARGET_NAME = 'sync-gem-demo-target'\n", + "DEMO_PROG = 'GS-2026A-T-999'\n", + "\n", + "# Create demo user and target (find-or-create to support re-runs).\n", + "demo_user, _ = get_user_model().objects.get_or_create(\n", + " username='sync-gem-demo-user',\n", + " defaults={'password': 'unused'},\n", + ")\n", + "demo_target = Target.objects.filter(name=DEMO_TARGET_NAME).first()\n", + "if demo_target is None:\n", + " demo_target = NonSiderealTargetFactory.create(name=DEMO_TARGET_NAME)\n", + "print(f'Demo user pk={demo_user.pk}, target pk={demo_target.pk}')" + ] + }, + { + "cell_type": "markdown", + "id": "e5f6a7b8", + "metadata": {}, + "source": [ + "## Create fixture ObservationRecords\n", + "\n", + "Four records to exercise the four scenarios. All use `password: '[redacted]'` as a harmless\n", + "placeholder (real submissions would contain an encrypted key here; the command strips it\n", + "before any processing — D-04 / GEM-SECURE-01)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "f6a7b8c9", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-27T05:09:11.865059Z", + "iopub.status.busy": "2026-06-27T05:09:11.864635Z", + "iopub.status.idle": "2026-06-27T05:09:11.919278Z", + "shell.execute_reply": "2026-06-27T05:09:11.918540Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-gem-demo-target @ GEM from None to PENDING\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-gem-demo-target @ GEM from None to PENDING\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-gem-demo-target @ GEM from None to PENDING\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-gem-demo-target @ GEM from None to PENDING\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created 4 ObservationRecords: [1, 2, 3, 4]\n" + ] + } + ], + "source": [ + "# Clean up any leftover demo records from prior runs.\n", + "ObservationRecord.objects.filter(user=demo_user, facility='GEM').delete()\n", + "CalendarEvent.objects.filter(url__startswith=f'GEM:{DEMO_PROG}/').delete()\n", + "\n", + "\n", + "def make_gem_record(obs_id: str, params: dict) -> ObservationRecord:\n", + " return ObservationRecord.objects.create(\n", + " observation_id=obs_id,\n", + " target=demo_target,\n", + " user=demo_user,\n", + " facility='GEM',\n", + " status='PENDING',\n", + " parameters=params,\n", + " )\n", + "\n", + "\n", + "# Scenario 1: explicit window (GEM-WINDOW-01)\n", + "rec_explicit = make_gem_record(\n", + " '9001',\n", + " {\n", + " 'prog': DEMO_PROG,\n", + " 'obsid': ['MM'],\n", + " 'ready': 'true',\n", + " 'password': '[redacted]',\n", + " 'windowDate': '2026-07-15',\n", + " 'windowTime': '02:30',\n", + " 'windowDuration': '4',\n", + " },\n", + ")\n", + "\n", + "# Scenario 2: Rap: derived window (GEM-WINDOW-02, QQ -> Rap:)\n", + "rec_rap = make_gem_record(\n", + " '9002',\n", + " {\n", + " 'prog': DEMO_PROG,\n", + " 'obsid': ['QQ'],\n", + " 'ready': 'true',\n", + " 'password': '[redacted]',\n", + " },\n", + ")\n", + "\n", + "# Scenario 3: Std: derived window (GEM-WINDOW-02, MM -> Std:)\n", + "rec_std = make_gem_record(\n", + " '9003',\n", + " {\n", + " 'prog': DEMO_PROG,\n", + " 'obsid': ['MM'],\n", + " 'ready': 'true',\n", + " 'password': '[redacted]',\n", + " },\n", + ")\n", + "\n", + "# Scenario 4: ON_HOLD (GEM-STATUS-01, ready='false')\n", + "rec_onhold = make_gem_record(\n", + " '9004',\n", + " {\n", + " 'prog': DEMO_PROG,\n", + " 'obsid': ['MM'],\n", + " 'ready': 'false',\n", + " 'password': '[redacted]',\n", + " },\n", + ")\n", + "\n", + "print('Created 4 ObservationRecords:', [rec_explicit.pk, rec_rap.pk, rec_std.pk, rec_onhold.pk])" + ] + }, + { + "cell_type": "markdown", + "id": "a7b8c9d0", + "metadata": {}, + "source": [ + "## Run sync command (first pass)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "b8c9d0e1", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-27T05:09:11.921729Z", + "iopub.status.busy": "2026-06-27T05:09:11.921361Z", + "iopub.status.idle": "2026-06-27T05:09:11.976358Z", + "shell.execute_reply": "2026-06-27T05:09:11.975635Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stdout: Gemini South: created: 4, updated: 0, unchanged: 0, skipped: 0\n", + "Gemini North: created: 0, updated: 0, unchanged: 0, skipped: 0\n", + "Done.\n", + "\n" + ] + } + ], + "source": [ + "import io\n", + "from datetime import timedelta\n", + "from datetime import timezone as dt_timezone\n", + "\n", + "from django.core.management import call_command\n", + "\n", + "stdout_buf = io.StringIO()\n", + "stderr_buf = io.StringIO()\n", + "call_command('sync_gemini_observation_calendar', stdout=stdout_buf, stderr=stderr_buf)\n", + "\n", + "print('stdout:', stdout_buf.getvalue())\n", + "if stderr_buf.getvalue():\n", + " print('stderr:', stderr_buf.getvalue())" + ] + }, + { + "cell_type": "markdown", + "id": "c9d0e1f2", + "metadata": {}, + "source": [ + "## Inspect created CalendarEvents" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "d0e1f2a3", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-27T05:09:11.979090Z", + "iopub.status.busy": "2026-06-27T05:09:11.978702Z", + "iopub.status.idle": "2026-06-27T05:09:12.012807Z", + "shell.execute_reply": "2026-06-27T05:09:12.011927Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total CalendarEvents for GS-2026A-T-999: 4\n", + "\n", + " url: GEM:GS-2026A-T-999/9001\n", + " title: Gemini South GMOS-S MOS ToO\n", + " telescope: Gemini South\n", + " instrument: GMOS-S MOS\n", + " proposal: GS-2026A-T-999\n", + " start_time: 2026-07-15 02:30:00+00:00\n", + " end_time: 2026-07-15 06:30:00+00:00\n", + "\n", + " url: GEM:GS-2026A-T-999/9002\n", + " title: Gemini South GMOS-S MOS ToO\n", + " telescope: Gemini South\n", + " instrument: GMOS-S MOS\n", + " proposal: GS-2026A-T-999\n", + " start_time: 2026-06-27 05:09:11.908810+00:00\n", + " end_time: 2026-06-28 05:09:11.908810+00:00\n", + "\n", + " url: GEM:GS-2026A-T-999/9003\n", + " title: Gemini South GMOS-S MOS ToO\n", + " telescope: Gemini South\n", + " instrument: GMOS-S MOS\n", + " proposal: GS-2026A-T-999\n", + " start_time: 2026-06-28 05:09:11.911586+00:00\n", + " end_time: 2026-07-04 05:09:11.911586+00:00\n", + "\n", + " url: GEM:GS-2026A-T-999/9004\n", + " title: [ON_HOLD] Gemini South GMOS-S MOS ToO\n", + " telescope: Gemini South\n", + " instrument: GMOS-S MOS\n", + " proposal: GS-2026A-T-999\n", + " start_time: 2026-06-28 05:09:11.914281+00:00\n", + " end_time: 2026-07-04 05:09:11.914281+00:00\n", + "\n" + ] + } + ], + "source": [ + "from datetime import datetime\n", + "\n", + "events = CalendarEvent.objects.filter(url__startswith=f'GEM:{DEMO_PROG}/').order_by('url')\n", + "print(f'Total CalendarEvents for {DEMO_PROG}: {events.count()}\\n')\n", + "for ev in events:\n", + " print(f' url: {ev.url}')\n", + " print(f' title: {ev.title}')\n", + " print(f' telescope: {ev.telescope}')\n", + " print(f' instrument: {ev.instrument}')\n", + " print(f' proposal: {ev.proposal}')\n", + " print(f' start_time: {ev.start_time}')\n", + " print(f' end_time: {ev.end_time}')\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Scenario 1: Explicit window (GEM-WINDOW-01)\n", + "\n", + "When `windowDate`/`windowTime`/`windowDuration` are present, the event's window is\n", + "derived directly from those values, not from the record's creation time." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "f2a3b4c5", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-27T05:09:12.015999Z", + "iopub.status.busy": "2026-06-27T05:09:12.015747Z", + "iopub.status.idle": "2026-06-27T05:09:12.048114Z", + "shell.execute_reply": "2026-06-27T05:09:12.047251Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Scenario 1 (explicit window) OK\n", + " start_time = 2026-07-15 02:30:00+00:00 (parsed from windowDate=2026-07-15, windowTime=02:30)\n", + " end_time = 2026-07-15 06:30:00+00:00 (start + 4h)\n" + ] + } + ], + "source": [ + "ev_explicit = CalendarEvent.objects.get(url=f'GEM:{DEMO_PROG}/9001')\n", + "expected_start = datetime(2026, 7, 15, 2, 30, tzinfo=dt_timezone.utc)\n", + "expected_end = expected_start + timedelta(hours=4)\n", + "\n", + "assert ev_explicit.start_time == expected_start, f'Expected {expected_start}, got {ev_explicit.start_time}'\n", + "assert ev_explicit.end_time == expected_end, f'Expected {expected_end}, got {ev_explicit.end_time}'\n", + "assert ev_explicit.instrument == 'GMOS-S MOS'\n", + "assert ev_explicit.telescope == 'Gemini South'\n", + "assert ev_explicit.proposal == DEMO_PROG\n", + "assert ev_explicit.title == 'Gemini South GMOS-S MOS ToO'\n", + "print('Scenario 1 (explicit window) OK')\n", + "print(f' start_time = {ev_explicit.start_time} (parsed from windowDate=2026-07-15, windowTime=02:30)')\n", + "print(f' end_time = {ev_explicit.end_time} (start + 4h)')" + ] + }, + { + "cell_type": "markdown", + "id": "a3b4c5d6", + "metadata": {}, + "source": [ + "## Scenario 2: Rapid ToO derived window (GEM-WINDOW-02, Rap:)\n", + "\n", + "When no explicit window is present and the obs code maps to a `'Rap: ...'` settings entry,\n", + "the window is `[record.created, record.created + 24h]`." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "b4c5d6e7", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-27T05:09:12.051427Z", + "iopub.status.busy": "2026-06-27T05:09:12.051157Z", + "iopub.status.idle": "2026-06-27T05:09:12.087514Z", + "shell.execute_reply": "2026-06-27T05:09:12.086674Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Scenario 2 (Rap: derived window) OK\n", + " start_time = 2026-06-27 05:09:11.908810+00:00 (== record.created)\n", + " end_time = 2026-06-28 05:09:11.908810+00:00 (created + 24h)\n" + ] + } + ], + "source": [ + "ev_rap = CalendarEvent.objects.get(url=f'GEM:{DEMO_PROG}/9002')\n", + "rec_rap.refresh_from_db() # ensure created timestamp is loaded\n", + "\n", + "assert ev_rap.start_time == rec_rap.created, f'Expected {rec_rap.created}, got {ev_rap.start_time}'\n", + "assert ev_rap.end_time == rec_rap.created + timedelta(hours=24)\n", + "assert ev_rap.instrument == 'GMOS-S MOS'\n", + "assert ev_rap.title == 'Gemini South GMOS-S MOS ToO'\n", + "print('Scenario 2 (Rap: derived window) OK')\n", + "print(f' start_time = {ev_rap.start_time} (== record.created)')\n", + "print(f' end_time = {ev_rap.end_time} (created + 24h)')" + ] + }, + { + "cell_type": "markdown", + "id": "c5d6e7f8", + "metadata": {}, + "source": [ + "## Scenario 3: Standard ToO derived window (GEM-WINDOW-02, Std:)\n", + "\n", + "When no explicit window is present and the obs code maps to a `'Std: ...'` settings entry,\n", + "the window is `[record.created + 24h, record.created + 7d]`." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "d6e7f8a9", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-27T05:09:12.090720Z", + "iopub.status.busy": "2026-06-27T05:09:12.090456Z", + "iopub.status.idle": "2026-06-27T05:09:12.123142Z", + "shell.execute_reply": "2026-06-27T05:09:12.122253Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Scenario 3 (Std: derived window) OK\n", + " start_time = 2026-06-28 05:09:11.911586+00:00 (created + 24h)\n", + " end_time = 2026-07-04 05:09:11.911586+00:00 (created + 7d)\n" + ] + } + ], + "source": [ + "ev_std = CalendarEvent.objects.get(url=f'GEM:{DEMO_PROG}/9003')\n", + "rec_std.refresh_from_db()\n", + "\n", + "assert ev_std.start_time == rec_std.created + timedelta(hours=24)\n", + "assert ev_std.end_time == rec_std.created + timedelta(days=7)\n", + "assert ev_std.instrument == 'GMOS-S MOS'\n", + "print('Scenario 3 (Std: derived window) OK')\n", + "print(f' start_time = {ev_std.start_time} (created + 24h)')\n", + "print(f' end_time = {ev_std.end_time} (created + 7d)')" + ] + }, + { + "cell_type": "markdown", + "id": "e7f8a9b0", + "metadata": {}, + "source": [ + "## Scenario 4: ON_HOLD title prefix (GEM-STATUS-01)\n", + "\n", + "When `ready='false'`, the event title is prefixed with `[ON_HOLD] `." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "f8a9b0c1", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-27T05:09:12.126573Z", + "iopub.status.busy": "2026-06-27T05:09:12.126269Z", + "iopub.status.idle": "2026-06-27T05:09:12.157484Z", + "shell.execute_reply": "2026-06-27T05:09:12.156607Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Scenario 4 (ON_HOLD) OK\n", + " title = '[ON_HOLD] Gemini South GMOS-S MOS ToO'\n" + ] + } + ], + "source": [ + "ev_onhold = CalendarEvent.objects.get(url=f'GEM:{DEMO_PROG}/9004')\n", + "\n", + "assert ev_onhold.title.startswith('[ON_HOLD] '), f'Expected [ON_HOLD] prefix, got: {ev_onhold.title!r}'\n", + "print('Scenario 4 (ON_HOLD) OK')\n", + "print(f' title = {ev_onhold.title!r}')" + ] + }, + { + "cell_type": "markdown", + "id": "a9b0c1d2", + "metadata": {}, + "source": [ + "## Scenario 4 (continued): Idempotent re-run (GEM-NOCHURN-01)\n", + "\n", + "Running the command again on unchanged records must leave `CalendarEvent.modified`\n", + "untouched and report `unchanged: 4`." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "b0c1d2e3", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-27T05:09:12.160674Z", + "iopub.status.busy": "2026-06-27T05:09:12.160432Z", + "iopub.status.idle": "2026-06-27T05:09:12.277662Z", + "shell.execute_reply": "2026-06-27T05:09:12.276715Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stdout (second run): Gemini South: created: 0, updated: 0, unchanged: 4, skipped: 0\n", + "Gemini North: created: 0, updated: 0, unchanged: 0, skipped: 0\n", + "Done.\n", + "\n", + "Scenario 4 (idempotent re-run) OK — modified timestamps all unchanged\n" + ] + } + ], + "source": [ + "# Snapshot modified timestamps before second run.\n", + "modified_before = {ev.pk: ev.modified for ev in events}\n", + "\n", + "stdout_buf2 = io.StringIO()\n", + "call_command('sync_gemini_observation_calendar', stdout=stdout_buf2, stderr=io.StringIO())\n", + "\n", + "print('stdout (second run):', stdout_buf2.getvalue())\n", + "\n", + "# Verify no new events were created.\n", + "assert CalendarEvent.objects.filter(url__startswith=f'GEM:{DEMO_PROG}/').count() == 4\n", + "\n", + "# Verify modified timestamps are unchanged.\n", + "for ev in CalendarEvent.objects.filter(url__startswith=f'GEM:{DEMO_PROG}/').order_by('url'):\n", + " assert (\n", + " ev.modified == modified_before[ev.pk]\n", + " ), f'Event pk={ev.pk} modified changed from {modified_before[ev.pk]} to {ev.modified}'\n", + "\n", + "assert 'unchanged: 4' in stdout_buf2.getvalue(), f'Expected \"unchanged: 4\" in: {stdout_buf2.getvalue()!r}'\n", + "print('Scenario 4 (idempotent re-run) OK — modified timestamps all unchanged')" + ] + }, + { + "cell_type": "markdown", + "id": "c1d2e3f4", + "metadata": {}, + "source": [ + "## Teardown\n", + "\n", + "Remove all demo fixtures so the notebook can be re-run cleanly." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "d2e3f4a5", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-27T05:09:12.280349Z", + "iopub.status.busy": "2026-06-27T05:09:12.280115Z", + "iopub.status.idle": "2026-06-27T05:09:12.323517Z", + "shell.execute_reply": "2026-06-27T05:09:12.322739Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Deleted 4 CalendarEvents and 4 ObservationRecords\n" + ] + } + ], + "source": [ + "deleted_events, _ = CalendarEvent.objects.filter(url__startswith=f'GEM:{DEMO_PROG}/').delete()\n", + "deleted_records, _ = ObservationRecord.objects.filter(user=demo_user, facility='GEM').delete()\n", + "print(f'Deleted {deleted_events} CalendarEvents and {deleted_records} ObservationRecords')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/notebooks/pre_executed/sync_lco_observation_calendar_demo.ipynb b/docs/notebooks/pre_executed/sync_lco_observation_calendar_demo.ipynb new file mode 100644 index 00000000..8d2e6a85 --- /dev/null +++ b/docs/notebooks/pre_executed/sync_lco_observation_calendar_demo.ipynb @@ -0,0 +1,1980 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3e1", + "metadata": {}, + "source": [ + "# LCO Queue Calendar Sync — Stage 3 Demo (sync_lco_observation_calendar)\n", + "\n", + "This notebook demonstrates `solsys_code/management/commands/sync_lco_observation_calendar.py`\n", + "(issue #37 Stage 3), the LCO queue sync command that syncs FTS/MuSCAT4 LCO queue\n", + "`ObservationRecord`s to the FOMO calendar as `CalendarEvent`s — transitioning from\n", + "a `[QUEUED]` scheduling-window banner to a clean placed block as the LCO scheduler\n", + "acts, with no-churn idempotency on unchanged records.\n", + "\n", + "It demonstrates:\n", + "\n", + "- Building a fixture `ObservationRecord` matching the shape used by\n", + " `solsys_code/tests/test_sync_lco_observation_calendar.py`\n", + "- Invoking `sync_lco_observation_calendar` via `call_command` while the record is\n", + " still unscheduled (queue window banner, `[QUEUED]` title prefix)\n", + "- Inspecting the resulting `[QUEUED]` `CalendarEvent` (times from\n", + " `parameters['start']`/`parameters['end']`, url from `LCOFacility().get_observation_url`)\n", + "- Simulating the LCO scheduler placing the observation (`scheduled_start`/`scheduled_end`\n", + " populated) and re-running the command to see the event transition to a clean\n", + " placed block\n", + "- Re-running the command a third time with no changes to confirm no-churn\n", + " idempotency (`modified` timestamp untouched, `unchanged: 1` in the summary)\n", + "\n", + "This notebook lives in `pre_executed/` because it is **DB-dependent** (it creates\n", + "an `ObservationRecord` fixture and `CalendarEvent` rows) and is therefore **NOT\n", + "run during Sphinx/CI/ReadTheDocs builds**, per `docs/notebooks/README.md`.\n" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4f2", + "metadata": {}, + "source": [ + "## Django setup\n", + "\n", + "Standard boilerplate to make `src.fomo.settings` importable from this notebook's\n", + "location (`docs/notebooks/pre_executed/` — three levels under the repo root, so\n", + "`parents[2]` gives the repo root) and to allow synchronous ORM calls inside\n", + "Jupyter's async event loop.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "c3d4e5a3", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:47.993492Z", + "iopub.status.busy": "2026-06-25T05:47:47.993029Z", + "iopub.status.idle": "2026-06-25T05:47:49.615349Z", + "shell.execute_reply": "2026-06-25T05:47:49.612918Z" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "import django\n", + "\n", + "# Ensure the repo root is on sys.path so `src.fomo.settings` is importable\n", + "# when this notebook is executed from docs/notebooks/pre_executed/.\n", + "# NOTE: parents[2] is correct only when the Jupyter kernel CWD is\n", + "# docs/notebooks/pre_executed/. Start Jupyter from that directory, or\n", + "# adjust the index if you launch from the repo root.\n", + "repo_root_path = Path.cwd().resolve().parents[2]\n", + "assert (repo_root_path / 'manage.py').exists(), (\n", + " f'Repo root not found at {repo_root_path}. '\n", + " 'Run Jupyter from docs/notebooks/pre_executed/ or adjust parents[] index.'\n", + ")\n", + "repo_root = str(repo_root_path)\n", + "if repo_root not in sys.path:\n", + " sys.path.insert(0, repo_root)\n", + "\n", + "os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'src.fomo.settings')\n", + "\n", + "# Jupyter's ipykernel runs inside an asyncio event loop, but Django's ORM is\n", + "# sync-only by default and refuses to run there; this opts back in.\n", + "os.environ.setdefault('DJANGO_ALLOW_ASYNC_UNSAFE', 'true')\n", + "\n", + "django.setup()" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6b4", + "metadata": {}, + "source": [ + "## Create a fixture ObservationRecord\n", + "\n", + "`sync_lco_observation_calendar` queries\n", + "`ObservationRecord(facility='LCO', parameters__proposal=)`. Here we build one\n", + "fixture record using the same `parameters` shape as\n", + "`solsys_code/tests/test_sync_lco_observation_calendar.py` (`proposal`, `start`,\n", + "`end`, `instrument_type`, `site` keys). The FK setup — a non-sidereal `Target` via\n", + "`NonSiderealTargetFactory` (FOMO targets are Solar System minor bodies, not\n", + "sidereal sources) and a `user` — mirrors that test's `setUpTestData`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "e5f6a7c5", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:49.620481Z", + "iopub.status.busy": "2026-06-25T05:47:49.619333Z", + "iopub.status.idle": "2026-06-25T05:47:50.018259Z", + "shell.execute_reply": "2026-06-25T05:47:50.014129Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "User sync-demo-user is not logged in. Cannot re-encrypt sensitive data. Clearing all encrypted fields instead.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Error clearing encrypted fields for model ESOProfile for user sync-demo-user: Model instances passed to related filters must be saved.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "No Profile found for sync-demo-user. Creating Profile.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Target post save hook: sync-demo-target created: True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Target post save hook: sync-demo-target created: False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ LCO from None to PENDING\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "observation_id : demo-900001\n", + "status : PENDING\n", + "scheduled_start : None\n", + "parameters : {'proposal': 'DEMOCODE', 'start': '2026-07-01T00:00:00', 'end': '2026-07-02T00:00:00', 'instrument_type': '2M0-SCICAM-MUSCAT', 'site': 'coj'}\n" + ] + } + ], + "source": [ + "from django.contrib.auth import get_user_model\n", + "from tom_observations.models import ObservationRecord\n", + "from tom_targets.models import Target\n", + "from tom_targets.tests.factories import NonSiderealTargetFactory\n", + "\n", + "DEMO_PROPOSAL = 'DEMOCODE'\n", + "DEMO_TARGET_NAME = 'sync-demo-target'\n", + "\n", + "# get_or_create / fixed-name lookup keep this cell re-runnable without\n", + "# accumulating a new Target (and its factory-generated extras/aliases) on\n", + "# every re-run. NonSiderealTargetFactory is used because FOMO targets are\n", + "# Solar System minor bodies (orbital elements), not sidereal sources.\n", + "demo_user, _ = get_user_model().objects.get_or_create(username='sync-demo-user')\n", + "demo_target = Target.objects.filter(name=DEMO_TARGET_NAME).first()\n", + "if demo_target is None:\n", + " demo_target = NonSiderealTargetFactory.create(name=DEMO_TARGET_NAME)\n", + "\n", + "record, created = ObservationRecord.objects.get_or_create(\n", + " observation_id='demo-900001',\n", + " defaults=dict(\n", + " target=demo_target,\n", + " user=demo_user,\n", + " facility='LCO',\n", + " status='PENDING',\n", + " scheduled_start=None,\n", + " scheduled_end=None,\n", + " parameters={\n", + " 'proposal': DEMO_PROPOSAL,\n", + " 'start': '2026-07-01T00:00:00',\n", + " 'end': '2026-07-02T00:00:00',\n", + " 'instrument_type': '2M0-SCICAM-MUSCAT',\n", + " 'site': 'coj',\n", + " },\n", + " ),\n", + ")\n", + "\n", + "print('observation_id :', record.observation_id)\n", + "print('status :', record.status)\n", + "print('scheduled_start :', record.scheduled_start)\n", + "print('parameters :', record.parameters)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "fb4e0a8b", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:50.023375Z", + "iopub.status.busy": "2026-06-25T05:47:50.022840Z", + "iopub.status.idle": "2026-06-25T05:47:50.106792Z", + "shell.execute_reply": "2026-06-25T05:47:50.103770Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'facility': 'LCO',\n", + " 'id': 91,\n", + " 'observation_id': 'demo-900001',\n", + " 'parameters': {'end': '2026-07-02T00:00:00',\n", + " 'instrument_type': '2M0-SCICAM-MUSCAT',\n", + " 'proposal': 'DEMOCODE',\n", + " 'site': 'coj',\n", + " 'start': '2026-07-01T00:00:00'},\n", + " 'scheduled_end': None,\n", + " 'scheduled_start': None,\n", + " 'status': 'PENDING',\n", + " 'target': 45,\n", + " 'user': 9}\n" + ] + } + ], + "source": [ + "from django.forms import model_to_dict\n", + "import pprint\n", + "\n", + "obsrecs = ObservationRecord.objects.all()\n", + "pprint.pprint(model_to_dict(obsrecs[0]))" + ] + }, + { + "cell_type": "markdown", + "id": "f6a7b8d6", + "metadata": {}, + "source": [ + "## Sync the queued (unscheduled) record\n", + "\n", + "`call_command` is the Django-recommended way to invoke management commands\n", + "programmatically. With `scheduled_start=None`, the command derives its times from\n", + "`parameters['start']`/`parameters['end']` and prefixes the title with `[QUEUED]`\n", + "(SYNC-02). The event's `url` is built via\n", + "`LCOFacility().get_observation_url(observation_id)` (SYNC-01).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a7b8c9e7", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:50.112512Z", + "iopub.status.busy": "2026-06-25T05:47:50.111484Z", + "iopub.status.idle": "2026-06-25T05:47:50.244474Z", + "shell.execute_reply": "2026-06-25T05:47:50.239396Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stdout: Done. proposal: DEMOCODE, LCO: created: 1, updated: 0, unchanged: 0, skipped: 0, extraction_failed: 0, telescope_api_failed: 0 | SOAR: created: 0, updated: 0, unchanged: 0, skipped: 0, extraction_failed: 0, telescope_api_failed: 0\n", + "\n" + ] + } + ], + "source": [ + "import io\n", + "\n", + "from django.core.management import call_command\n", + "\n", + "stdout_buf = io.StringIO()\n", + "stderr_buf = io.StringIO()\n", + "\n", + "call_command('sync_lco_observation_calendar', '--proposal', DEMO_PROPOSAL, stdout=stdout_buf, stderr=stderr_buf)\n", + "\n", + "print('stdout:', stdout_buf.getvalue())\n", + "if stderr_buf.getvalue():\n", + " print('stderr:', stderr_buf.getvalue())" + ] + }, + { + "cell_type": "markdown", + "id": "b8c9d0e8", + "metadata": {}, + "source": [ + "## Inspect the queued CalendarEvent\n", + "\n", + "The resulting event has a `[QUEUED]`-prefixed title, times taken from the\n", + "parameters window, and a url built from the LCO portal (containing `/requests/`).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "c9d0e1f9", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:50.249958Z", + "iopub.status.busy": "2026-06-25T05:47:50.249354Z", + "iopub.status.idle": "2026-06-25T05:47:50.338329Z", + "shell.execute_reply": "2026-06-25T05:47:50.331696Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "title : [QUEUED] 2m0 2M0-SCICAM-MUSCAT\n", + "start_time : 2026-07-01T00:00:00+00:00\n", + "end_time : 2026-07-02T00:00:00+00:00\n", + "url : https://observe.lco.global/requests/demo-900001\n", + "telescope : 2m0\n", + "instrument : 2M0-SCICAM-MUSCAT\n", + "proposal : DEMOCODE\n", + "description :\n", + " Proposal: DEMOCODE\n", + " Status: PENDING\n", + " Window (UTC): 2026-07-01T00:00:00 to 2026-07-02T00:00:00\n" + ] + } + ], + "source": [ + "from tom_calendar.models import CalendarEvent\n", + "from tom_observations.facilities.lco import LCOFacility\n", + "\n", + "event_url = LCOFacility().get_observation_url('demo-900001')\n", + "event = CalendarEvent.objects.get(url=event_url)\n", + "\n", + "print('title :', event.title) # starts with '[QUEUED]'\n", + "print('start_time :', event.start_time.isoformat())\n", + "print('end_time :', event.end_time.isoformat())\n", + "print('url :', event.url) # contains '/requests/'\n", + "print('telescope :', event.telescope)\n", + "print('instrument :', event.instrument)\n", + "print('proposal :', event.proposal)\n", + "print('description :')\n", + "for line in event.description.splitlines():\n", + " print(' ', line)" + ] + }, + { + "cell_type": "markdown", + "id": "aa42da31", + "metadata": {}, + "source": [ + "## Check all other CalendarEvent's\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "67e6f772", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:50.343397Z", + "iopub.status.busy": "2026-06-25T05:47:50.343010Z", + "iopub.status.idle": "2026-06-25T05:47:50.421901Z", + "shell.execute_reply": "2026-06-25T05:47:50.419065Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NTT EFOSC2 2026-07-09T22:06:35+00:00->2026-07-10T11:29:46+00:00 NTT+EFOSC2 \n", + "NTT EFOSC2 2026-07-10T22:07:04+00:00->2026-07-11T11:29:34+00:00 NTT+EFOSC2 \n", + "NTT EFOSC2 2026-07-11T22:07:32+00:00->2026-07-12T11:29:21+00:00 NTT+EFOSC2 \n", + "NTT EFOSC2 2026-07-12T22:08:02+00:00->2026-07-13T11:29:06+00:00 NTT+EFOSC2 \n", + "NTT EFOSC2 2026-07-13T22:08:31+00:00->2026-07-14T11:28:50+00:00 NTT+EFOSC2 \n", + "FTS MUSCAT4 2026-07-10T07:21:08+00:00->2026-07-10T20:56:59+00:00 FTS+MUSCAT4 \n", + "FTS MUSCAT4 2026-07-11T07:21:39+00:00->2026-07-11T20:56:44+00:00 FTS+MUSCAT4 \n", + "FTS MUSCAT4 2026-07-12T07:22:10+00:00->2026-07-12T20:56:28+00:00 FTS+MUSCAT4 \n", + "[EXPIRED] 1m0 1M0-SCICAM-SINISTRO 2026-06-15T00:00:00+00:00->2026-06-16T00:00:00+00:00 1m0+1M0-SCICAM-SINISTRO LTP2025A-004\n", + "[UNVERIFIED] 1m0 1M0-SCICAM-SINISTRO 2026-06-24T18:20:00+00:00->2026-06-24T18:36:51+00:00 1m0+1M0-SCICAM-SINISTRO LTP2025A-004\n", + "[QUEUED] 2m0 2M0-SCICAM-MUSCAT 2026-07-01T00:00:00+00:00->2026-07-02T00:00:00+00:00 2m0+2M0-SCICAM-MUSCAT DEMOCODE\n" + ] + } + ], + "source": [ + "events = CalendarEvent.objects.all()\n", + "for event in events:\n", + " print(\n", + " f'{event.title} {event.start_time.isoformat()}->{event.end_time.isoformat()} {event.telescope}+{event.instrument} {event.proposal}'\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "d0e1f2a0", + "metadata": {}, + "source": [ + "## Simulate the LCO scheduler placing the observation\n", + "\n", + "Setting `scheduled_start`/`scheduled_end` on the record and re-running the command\n", + "transitions the event to the clean placed form — times now come from the\n", + "scheduled fields and the `[QUEUED]` prefix drops (SYNC-03).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "e1f2a3b1", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:50.431541Z", + "iopub.status.busy": "2026-06-25T05:47:50.430897Z", + "iopub.status.idle": "2026-06-25T05:47:51.058037Z", + "shell.execute_reply": "2026-06-25T05:47:51.055648Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "title : [UNVERIFIED] 2m0 2M0-SCICAM-MUSCAT\n", + "start_time : 2026-07-05T10:00:00+00:00\n", + "end_time : 2026-07-05T12:00:00+00:00\n" + ] + } + ], + "source": [ + "from datetime import datetime\n", + "from datetime import timezone as dt_timezone\n", + "\n", + "record.scheduled_start = datetime(2026, 7, 5, 10, 0, 0, tzinfo=dt_timezone.utc)\n", + "record.scheduled_end = datetime(2026, 7, 5, 12, 0, 0, tzinfo=dt_timezone.utc)\n", + "record.save()\n", + "\n", + "call_command(\n", + " 'sync_lco_observation_calendar',\n", + " '--proposal',\n", + " DEMO_PROPOSAL,\n", + " stdout=io.StringIO(),\n", + " stderr=io.StringIO(),\n", + ")\n", + "\n", + "event = CalendarEvent.objects.get(url=event_url)\n", + "print('title :', event.title) # no '[QUEUED]' prefix now\n", + "print('start_time :', event.start_time.isoformat())\n", + "print('end_time :', event.end_time.isoformat())" + ] + }, + { + "cell_type": "markdown", + "id": "f2a3b4c2", + "metadata": {}, + "source": [ + "## No-churn idempotency\n", + "\n", + "Re-running the command a third time with no changes leaves the event's `modified`\n", + "timestamp unchanged, and the stdout summary reports `unchanged: 1` (SYNC-04).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a3b4c5d3", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:51.062777Z", + "iopub.status.busy": "2026-06-25T05:47:51.062390Z", + "iopub.status.idle": "2026-06-25T05:47:51.414971Z", + "shell.execute_reply": "2026-06-25T05:47:51.412750Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "modified unchanged: True\n", + "stdout: Done. proposal: DEMOCODE, LCO: created: 0, updated: 0, unchanged: 1, skipped: 0, extraction_failed: 0, telescope_api_failed: 1 | SOAR: created: 0, updated: 0, unchanged: 0, skipped: 0, extraction_failed: 0, telescope_api_failed: 0\n", + "\n" + ] + } + ], + "source": [ + "modified_before = CalendarEvent.objects.get(url=event_url).modified\n", + "\n", + "stdout_buf3 = io.StringIO()\n", + "call_command(\n", + " 'sync_lco_observation_calendar',\n", + " '--proposal',\n", + " DEMO_PROPOSAL,\n", + " stdout=stdout_buf3,\n", + " stderr=io.StringIO(),\n", + ")\n", + "\n", + "event = CalendarEvent.objects.get(url=event_url)\n", + "print('modified unchanged:', event.modified == modified_before)\n", + "print('stdout:', stdout_buf3.getvalue()) # shows 'unchanged: 1'" + ] + }, + { + "cell_type": "markdown", + "id": "p5a1b2c301", + "metadata": {}, + "source": [ + "## Phase 5: multi-proposal and multi-facility sync (LCO + SOAR)\n", + "\n", + "Phase 05 (v1.3) generalized `sync_lco_observation_calendar` so `--proposal` accepts\n", + "a comma-separated list of codes or the case-insensitive `ALL` token (SELECT-02/03),\n", + "and a single run now covers both `facility='LCO'` and `facility='SOAR'`\n", + "`ObservationRecord`s, each dispatched through its own facility instance\n", + "(SELECT-04/05).\n", + "\n", + "The cells below create their own fixture records with distinct `observation_id`s\n", + "(prefixed `demo-60xxxx`) so they do not collide with the Phase-4 `demo-900001`\n", + "record created above. They reuse the existing `demo_target`/`demo_user` fixtures\n", + "created earlier in this notebook. All Phase-5 fixtures are removed by the\n", + "teardown cell at the end of the notebook.\n" + ] + }, + { + "cell_type": "markdown", + "id": "p5b2c3d402", + "metadata": {}, + "source": [ + "### SELECT-02: comma-list `--proposal` matches exactly the listed codes\n", + "\n", + "Four LCO fixture records are created with proposals `PHASE5-A`, `PHASE5-B`,\n", + "`PHASE5-C`, and a decoy `PHASE5-AB` (a superstring of `PHASE5-A`/`PHASE5-B`, to\n", + "prove the match is exact-code, not substring). Running with\n", + "`--proposal PHASE5-A,PHASE5-B` must create `CalendarEvent`s for the A and B\n", + "records only — **not** for `PHASE5-C` (unselected) and **not** for the\n", + "`PHASE5-AB` decoy (substring-safe `__in` matching).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "p5b2c3d403", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:51.419561Z", + "iopub.status.busy": "2026-06-25T05:47:51.419216Z", + "iopub.status.idle": "2026-06-25T05:47:51.633365Z", + "shell.execute_reply": "2026-06-25T05:47:51.629446Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ LCO from None to PENDING\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ LCO from None to PENDING\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ LCO from None to PENDING\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ LCO from None to PENDING\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PASS: demo-602001 has event\n", + "PASS: demo-602002 has event\n", + "PASS: demo-602003 has no event\n", + "PASS: demo-602004 has no event\n" + ] + } + ], + "source": [ + "demo_phase5_select02_ids = ['demo-602001', 'demo-602002', 'demo-602003', 'demo-602004']\n", + "select02_proposals = ['PHASE5-A', 'PHASE5-B', 'PHASE5-C', 'PHASE5-AB']\n", + "\n", + "for observation_id, proposal in zip(demo_phase5_select02_ids, select02_proposals, strict=True):\n", + " ObservationRecord.objects.get_or_create(\n", + " observation_id=observation_id,\n", + " defaults=dict(\n", + " target=demo_target,\n", + " user=demo_user,\n", + " facility='LCO',\n", + " status='PENDING',\n", + " scheduled_start=None,\n", + " scheduled_end=None,\n", + " parameters={\n", + " 'proposal': proposal,\n", + " 'start': '2026-07-01T00:00:00',\n", + " 'end': '2026-07-02T00:00:00',\n", + " 'instrument_type': '2M0-SCICAM-MUSCAT',\n", + " 'site': 'coj',\n", + " },\n", + " ),\n", + " )\n", + "\n", + "call_command(\n", + " 'sync_lco_observation_calendar',\n", + " '--proposal',\n", + " 'PHASE5-A,PHASE5-B',\n", + " stdout=io.StringIO(),\n", + " stderr=io.StringIO(),\n", + ")\n", + "\n", + "# Selected codes (A, B) should have events; unselected C and decoy AB should not.\n", + "selected_ids = ['demo-602001', 'demo-602002']\n", + "unselected_ids = ['demo-602003', 'demo-602004'] # C, decoy AB\n", + "\n", + "for observation_id in selected_ids:\n", + " url = LCOFacility().get_observation_url(observation_id)\n", + " has_event = CalendarEvent.objects.filter(url=url).exists()\n", + " print(f'PASS: {observation_id} has event' if has_event else f'FAIL: {observation_id} missing event')\n", + "\n", + "for observation_id in unselected_ids:\n", + " url = LCOFacility().get_observation_url(observation_id)\n", + " has_event = CalendarEvent.objects.filter(url=url).exists()\n", + " print(f'PASS: {observation_id} has no event' if not has_event else f'FAIL: {observation_id} unexpectedly has event')" + ] + }, + { + "cell_type": "markdown", + "id": "p5c3d4e504", + "metadata": {}, + "source": [ + "### SELECT-03: `--proposal all` (lowercase) syncs every record regardless of proposal\n", + "\n", + "A fresh fixture with a new, previously-unselected proposal (`PHASE5-D`) and a\n", + "`facility='SOAR'` fixture (`PHASE5-SOAR`) are added. Running with\n", + "`--proposal all` (lowercase, demonstrating case-insensitivity of the `ALL`\n", + "sentinel) must sync **every** Phase-5 fixture record created so far — including\n", + "the previously-unselected `PHASE5-C`, the decoy `PHASE5-AB`, the new\n", + "`PHASE5-D`, and the SOAR record — because the `ALL` token drops the proposal\n", + "filter entirely.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "p5c3d4e505", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:51.638947Z", + "iopub.status.busy": "2026-06-25T05:47:51.638009Z", + "iopub.status.idle": "2026-06-25T05:47:55.544891Z", + "shell.execute_reply": "2026-06-25T05:47:55.542503Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ LCO from None to PENDING\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ SOAR from None to PENDING\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PASS: demo-602001 has event\n", + "PASS: demo-602002 has event\n", + "PASS: demo-602003 has event\n", + "PASS: demo-602004 has event\n", + "PASS: demo-603001 has event\n", + "PASS: demo-603002 has event\n" + ] + } + ], + "source": [ + "ObservationRecord.objects.get_or_create(\n", + " observation_id='demo-603001',\n", + " defaults=dict(\n", + " target=demo_target,\n", + " user=demo_user,\n", + " facility='LCO',\n", + " status='PENDING',\n", + " scheduled_start=None,\n", + " scheduled_end=None,\n", + " parameters={\n", + " 'proposal': 'PHASE5-D',\n", + " 'start': '2026-07-01T00:00:00',\n", + " 'end': '2026-07-02T00:00:00',\n", + " 'instrument_type': '2M0-SCICAM-MUSCAT',\n", + " 'site': 'coj',\n", + " },\n", + " ),\n", + ")\n", + "\n", + "ObservationRecord.objects.get_or_create(\n", + " observation_id='demo-603002',\n", + " defaults=dict(\n", + " target=demo_target,\n", + " user=demo_user,\n", + " facility='SOAR',\n", + " status='PENDING',\n", + " scheduled_start=None,\n", + " scheduled_end=None,\n", + " parameters={\n", + " 'proposal': 'PHASE5-SOAR',\n", + " 'start': '2026-07-01T00:00:00',\n", + " 'end': '2026-07-02T00:00:00',\n", + " 'instrument_type': 'SOAR_GHTS_REDCAM',\n", + " 'site': 'sor',\n", + " },\n", + " ),\n", + ")\n", + "\n", + "call_command(\n", + " 'sync_lco_observation_calendar',\n", + " '--proposal',\n", + " 'all',\n", + " stdout=io.StringIO(),\n", + " stderr=io.StringIO(),\n", + ")\n", + "\n", + "# Every Phase-5 fixture created so far should now have a CalendarEvent.\n", + "all_phase5_ids_so_far = demo_phase5_select02_ids + ['demo-603001', 'demo-603002']\n", + "for observation_id in all_phase5_ids_so_far:\n", + " url = LCOFacility().get_observation_url(observation_id)\n", + " has_event = CalendarEvent.objects.filter(url=url).exists()\n", + " print(f'PASS: {observation_id} has event' if has_event else f'FAIL: {observation_id} missing event')" + ] + }, + { + "cell_type": "markdown", + "id": "p5d4e5f606", + "metadata": {}, + "source": [ + "### SELECT-04 / D-08: a single run covers both LCO and SOAR, with a per-facility summary\n", + "\n", + "One fresh LCO fixture and one fresh SOAR fixture share the same proposal\n", + "(`PHASE5-BOTH`). A single `call_command` invocation must produce\n", + "`CalendarEvent`s for **both** records (SELECT-04). The captured stdout shows\n", + "separate `LCO:` and `SOAR:` created/updated/unchanged/skipped counts (D-08) —\n", + "the visible side effect of the command's multi-facility dispatch.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "p5d4e5f607", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:55.551974Z", + "iopub.status.busy": "2026-06-25T05:47:55.551209Z", + "iopub.status.idle": "2026-06-25T05:47:55.714547Z", + "shell.execute_reply": "2026-06-25T05:47:55.712595Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ LCO from None to PENDING\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ SOAR from None to PENDING\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "demo-604002 facility: SOAR\n", + "PASS: demo-604001 (LCO) has event\n", + "PASS: demo-604002 (SOAR) has event\n", + "stdout: Done. proposal: PHASE5-BOTH, LCO: created: 1, updated: 0, unchanged: 0, skipped: 0, extraction_failed: 0, telescope_api_failed: 0 | SOAR: created: 1, updated: 0, unchanged: 0, skipped: 0, extraction_failed: 0, telescope_api_failed: 0\n", + "\n" + ] + } + ], + "source": [ + "ObservationRecord.objects.get_or_create(\n", + " observation_id='demo-604001',\n", + " defaults=dict(\n", + " target=demo_target,\n", + " user=demo_user,\n", + " facility='LCO',\n", + " status='PENDING',\n", + " scheduled_start=None,\n", + " scheduled_end=None,\n", + " parameters={\n", + " 'proposal': 'PHASE5-BOTH',\n", + " 'start': '2026-07-01T00:00:00',\n", + " 'end': '2026-07-02T00:00:00',\n", + " 'instrument_type': '2M0-SCICAM-MUSCAT',\n", + " 'site': 'coj',\n", + " },\n", + " ),\n", + ")\n", + "\n", + "ObservationRecord.objects.get_or_create(\n", + " observation_id='demo-604002',\n", + " defaults=dict(\n", + " target=demo_target,\n", + " user=demo_user,\n", + " facility='SOAR',\n", + " status='PENDING',\n", + " scheduled_start=None,\n", + " scheduled_end=None,\n", + " parameters={\n", + " 'proposal': 'PHASE5-BOTH',\n", + " 'start': '2026-07-01T00:00:00',\n", + " 'end': '2026-07-02T00:00:00',\n", + " 'instrument_type': 'SOAR_GHTS_REDCAM',\n", + " 'site': 'sor',\n", + " },\n", + " ),\n", + ")\n", + "\n", + "# Guard against a silently-wrong fixture: confirm facility persisted as 'SOAR'.\n", + "print('demo-604002 facility:', ObservationRecord.objects.get(observation_id='demo-604002').facility)\n", + "\n", + "select04_stdout = io.StringIO()\n", + "call_command(\n", + " 'sync_lco_observation_calendar',\n", + " '--proposal',\n", + " 'PHASE5-BOTH',\n", + " stdout=select04_stdout,\n", + " stderr=io.StringIO(),\n", + ")\n", + "\n", + "lco_url = LCOFacility().get_observation_url('demo-604001')\n", + "soar_url = LCOFacility().get_observation_url('demo-604002')\n", + "lco_has_event = CalendarEvent.objects.filter(url=lco_url).exists()\n", + "soar_has_event = CalendarEvent.objects.filter(url=soar_url).exists()\n", + "print('PASS: demo-604001 (LCO) has event' if lco_has_event else 'FAIL: demo-604001 missing event')\n", + "print('PASS: demo-604002 (SOAR) has event' if soar_has_event else 'FAIL: demo-604002 missing event')\n", + "\n", + "print('stdout:', select04_stdout.getvalue()) # note separate LCO: / SOAR: counts (D-08)" + ] + }, + { + "cell_type": "markdown", + "id": "p5e5f6a708", + "metadata": {}, + "source": [ + "### SELECT-05: not demonstrated here (verified by a discriminating spy test)\n", + "\n", + "SELECT-05 requires proving that each `ObservationRecord` is dispatched through\n", + "the facility instance matching its own `facility` value — i.e. a `SOAR` record\n", + "is never processed via a reused `LCOFacility` instance. This cannot be\n", + "meaningfully demonstrated in this notebook: `LCOFacility().get_observation_url()`\n", + "and `SOARFacility().get_observation_url()` return byte-identical strings, so a\n", + "black-box check of the resulting `CalendarEvent.url`/fields cannot discriminate\n", + "which class actually handled the record. Proving SELECT-05 requires patching\n", + "both facility classes' methods and asserting which one is called (a\n", + "discriminating spy) — not appropriate for this notebook's `call_command`-only\n", + "style.\n", + "\n", + "SELECT-05 is instead verified by\n", + "`test_select_05_soar_record_uses_soar_facility_instance` in\n", + "`solsys_code/tests/test_sync_lco_observation_calendar.py`.\n" + ] + }, + { + "cell_type": "markdown", + "id": "p6a1b2c601", + "metadata": {}, + "source": [ + "## Phase 6: correct instrument-type extraction (EXTRACT-01/EXTRACT-02)\n", + "\n", + "Phase 06 (v1.3) replaced the flat `parameters['instrument_type']` read with a\n", + "`c_1..c_5` multi-config scanner (`_extract_instrument`) that distinguishes a\n", + "record's scientifically meaningful configuration from SOAR's calibration\n", + "configs (`ARC`/`LAMP_FLAT`) and detects LCO MUSCAT's per-channel exposure\n", + "shape (no flat `c_N_exposure_time` key). It also introduces a dedicated\n", + "`extraction_failed` counter, distinct from `skipped`, for records where no\n", + "instrument can be determined at all (D-06).\n", + "\n", + "The cells below create their own fixture records with distinct\n", + "`observation_id`s (prefixed `demo-606xxx`), separate from the Phase-4/5 demos\n", + "above, and are removed by the teardown cell at the end of the notebook." + ] + }, + { + "cell_type": "markdown", + "id": "p6b2c3d602", + "metadata": {}, + "source": [ + "### EXTRACT-02: SOAR multi-config — SPECTRUM picked over ARC/LAMP_FLAT calibration configs\n", + "\n", + "A SOAR fixture record carries three multi-configs: `c_1` (`SPECTRUM`,\n", + "`SOAR_GHTS_REDCAM` — the science config), `c_2` (`ARC`,\n", + "`SOAR_GHTS_REDCAM_ARC` — a calibration config), and `c_3` (`LAMP_FLAT`,\n", + "`SOAR_GHTS_REDCAM_LAMPFLAT` — another calibration config), plus a decoy flat\n", + "`instrument_type='NOT-THE-SOURCE'` that must be ignored. The resulting\n", + "`CalendarEvent.instrument` must be the SPECTRUM config's value, never either\n", + "calibration config's value." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "p6b2c3d603", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:55.719131Z", + "iopub.status.busy": "2026-06-25T05:47:55.718679Z", + "iopub.status.idle": "2026-06-25T05:47:55.855100Z", + "shell.execute_reply": "2026-06-25T05:47:55.853314Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ SOAR from None to PENDING\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "event.instrument: SOAR_GHTS_REDCAM\n", + "PASS: SPECTRUM config instrument extracted\n", + "PASS: calibration config instrument (ARC/LAMP_FLAT) not used\n" + ] + } + ], + "source": [ + "ObservationRecord.objects.get_or_create(\n", + " observation_id='demo-606001',\n", + " defaults=dict(\n", + " target=demo_target,\n", + " user=demo_user,\n", + " facility='SOAR',\n", + " status='PENDING',\n", + " scheduled_start=None,\n", + " scheduled_end=None,\n", + " parameters={\n", + " 'proposal': 'PHASE6-SOAR',\n", + " 'start': '2026-07-01T00:00:00',\n", + " 'end': '2026-07-02T00:00:00',\n", + " 'instrument_type': 'NOT-THE-SOURCE',\n", + " 'site': 'sor',\n", + " 'c_1_configuration_type': 'SPECTRUM',\n", + " 'c_1_instrument_type': 'SOAR_GHTS_REDCAM',\n", + " 'c_2_configuration_type': 'ARC',\n", + " 'c_2_instrument_type': 'SOAR_GHTS_REDCAM_ARC',\n", + " 'c_3_configuration_type': 'LAMP_FLAT',\n", + " 'c_3_instrument_type': 'SOAR_GHTS_REDCAM_LAMPFLAT',\n", + " },\n", + " ),\n", + ")\n", + "\n", + "call_command(\n", + " 'sync_lco_observation_calendar',\n", + " '--proposal',\n", + " 'PHASE6-SOAR',\n", + " stdout=io.StringIO(),\n", + " stderr=io.StringIO(),\n", + ")\n", + "\n", + "soar_url = LCOFacility().get_observation_url('demo-606001')\n", + "event = CalendarEvent.objects.get(url=soar_url)\n", + "print('event.instrument:', event.instrument)\n", + "\n", + "if event.instrument == 'SOAR_GHTS_REDCAM':\n", + " print('PASS: SPECTRUM config instrument extracted')\n", + "else:\n", + " print('FAIL: expected SOAR_GHTS_REDCAM, got', event.instrument)\n", + "\n", + "if event.instrument not in ('SOAR_GHTS_REDCAM_ARC', 'SOAR_GHTS_REDCAM_LAMPFLAT'):\n", + " print('PASS: calibration config instrument (ARC/LAMP_FLAT) not used')\n", + "else:\n", + " print('FAIL: calibration config instrument leaked through:', event.instrument)" + ] + }, + { + "cell_type": "markdown", + "id": "p6c3d4e604", + "metadata": {}, + "source": [ + "### EXTRACT-02: LCO MUSCAT per-channel exposure — no flat c_N_exposure_time key\n", + "\n", + "An LCO fixture record carries `c_1` with `configuration_type='EXPOSE'` and\n", + "`instrument_type='2M0-SCICAM-MUSCAT'`, but no flat `c_1_exposure_time` key —\n", + "only the four MUSCAT per-channel keys `c_1_ic_1_exposure_time_{g,r,i,z}`. The\n", + "extraction must still resolve `c_1` via the per-channel exposure-signal\n", + "fallback path (D-04), not via a flat exposure key that doesn't exist here." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "p6c3d4e605", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:55.859816Z", + "iopub.status.busy": "2026-06-25T05:47:55.859200Z", + "iopub.status.idle": "2026-06-25T05:47:56.400386Z", + "shell.execute_reply": "2026-06-25T05:47:56.398075Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ LCO from None to PENDING\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "event.instrument: 2M0-SCICAM-MUSCAT\n", + "PASS: MUSCAT per-channel exposure signal extracted instrument\n" + ] + } + ], + "source": [ + "ObservationRecord.objects.get_or_create(\n", + " observation_id='demo-606002',\n", + " defaults=dict(\n", + " target=demo_target,\n", + " user=demo_user,\n", + " facility='LCO',\n", + " status='PENDING',\n", + " scheduled_start=None,\n", + " scheduled_end=None,\n", + " parameters={\n", + " 'proposal': 'PHASE6-MUSCAT',\n", + " 'start': '2026-07-01T00:00:00',\n", + " 'end': '2026-07-02T00:00:00',\n", + " 'instrument_type': 'NOT-THE-SOURCE',\n", + " 'site': 'coj',\n", + " 'c_1_configuration_type': 'EXPOSE',\n", + " 'c_1_instrument_type': '2M0-SCICAM-MUSCAT',\n", + " 'c_1_ic_1_exposure_time_g': 30.0,\n", + " 'c_1_ic_1_exposure_time_r': 30.0,\n", + " 'c_1_ic_1_exposure_time_i': 30.0,\n", + " 'c_1_ic_1_exposure_time_z': 30.0,\n", + " },\n", + " ),\n", + ")\n", + "\n", + "call_command(\n", + " 'sync_lco_observation_calendar',\n", + " '--proposal',\n", + " 'PHASE6-MUSCAT',\n", + " stdout=io.StringIO(),\n", + " stderr=io.StringIO(),\n", + ")\n", + "\n", + "muscat_url = LCOFacility().get_observation_url('demo-606002')\n", + "event = CalendarEvent.objects.get(url=muscat_url)\n", + "print('event.instrument:', event.instrument)\n", + "\n", + "if event.instrument == '2M0-SCICAM-MUSCAT':\n", + " print('PASS: MUSCAT per-channel exposure signal extracted instrument')\n", + "else:\n", + " print('FAIL: expected 2M0-SCICAM-MUSCAT, got', event.instrument)" + ] + }, + { + "cell_type": "markdown", + "id": "p6d4e5f606", + "metadata": {}, + "source": [ + "### D-06: fully-malformed record skipped and counted under extraction_failed\n", + "\n", + "One malformed LCO fixture (`demo-606003`) has `c_1_configuration_type='ARC'`\n", + "(a calibration-only type, never in the science whitelist), no exposure-time\n", + "keys at all, and an explicit `instrument_type=None` override so even the flat\n", + "fallback is unavailable — no signal anywhere lets `_extract_instrument`\n", + "resolve an instrument. A baseline-good sibling fixture (`demo-606004`, default\n", + "shape, same proposal) proves the malformed record's failure doesn't block the\n", + "good one in the same `call_command` invocation.\n", + "\n", + "The printed run summary shows `extraction_failed: 1` — a counter distinct\n", + "from `skipped` — and stderr logs the malformed record's `observation_id` for\n", + "diagnosis." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "p6d4e5f607", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:56.405425Z", + "iopub.status.busy": "2026-06-25T05:47:56.404950Z", + "iopub.status.idle": "2026-06-25T05:47:56.540365Z", + "shell.execute_reply": "2026-06-25T05:47:56.537769Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ LCO from None to PENDING\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ LCO from None to PENDING\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stdout: Done. proposal: PHASE6-MALFORMED, LCO: created: 1, updated: 0, unchanged: 0, skipped: 0, extraction_failed: 1, telescope_api_failed: 0 | SOAR: created: 0, updated: 0, unchanged: 0, skipped: 0, extraction_failed: 0, telescope_api_failed: 0\n", + "\n", + "stderr: Skipping observation_id='demo-606003': No recognized configuration_type or exposure signal found in observation_id='demo-606003' parameters\n", + "\n", + "PASS: demo-606004 (good) has event\n", + "PASS: demo-606003 (malformed) has no event\n" + ] + } + ], + "source": [ + "ObservationRecord.objects.get_or_create(\n", + " observation_id='demo-606003',\n", + " defaults=dict(\n", + " target=demo_target,\n", + " user=demo_user,\n", + " facility='LCO',\n", + " status='PENDING',\n", + " scheduled_start=None,\n", + " scheduled_end=None,\n", + " parameters={\n", + " 'proposal': 'PHASE6-MALFORMED',\n", + " 'start': '2026-07-01T00:00:00',\n", + " 'end': '2026-07-02T00:00:00',\n", + " 'instrument_type': None,\n", + " 'site': 'coj',\n", + " 'c_1_configuration_type': 'ARC',\n", + " 'c_1_instrument_type': 'SOMETHING',\n", + " },\n", + " ),\n", + ")\n", + "\n", + "ObservationRecord.objects.get_or_create(\n", + " observation_id='demo-606004',\n", + " defaults=dict(\n", + " target=demo_target,\n", + " user=demo_user,\n", + " facility='LCO',\n", + " status='PENDING',\n", + " scheduled_start=None,\n", + " scheduled_end=None,\n", + " parameters={\n", + " 'proposal': 'PHASE6-MALFORMED',\n", + " 'start': '2026-07-01T00:00:00',\n", + " 'end': '2026-07-02T00:00:00',\n", + " 'instrument_type': '2M0-SCICAM-MUSCAT',\n", + " 'site': 'ogg',\n", + " },\n", + " ),\n", + ")\n", + "\n", + "malformed_stdout = io.StringIO()\n", + "malformed_stderr = io.StringIO()\n", + "call_command(\n", + " 'sync_lco_observation_calendar',\n", + " '--proposal',\n", + " 'PHASE6-MALFORMED',\n", + " stdout=malformed_stdout,\n", + " stderr=malformed_stderr,\n", + ")\n", + "\n", + "print('stdout:', malformed_stdout.getvalue())\n", + "print('stderr:', malformed_stderr.getvalue())\n", + "\n", + "good_url = LCOFacility().get_observation_url('demo-606004')\n", + "bad_url = LCOFacility().get_observation_url('demo-606003')\n", + "good_has_event = CalendarEvent.objects.filter(url=good_url).exists()\n", + "bad_has_event = CalendarEvent.objects.filter(url=bad_url).exists()\n", + "\n", + "print('PASS: demo-606004 (good) has event' if good_has_event else 'FAIL: demo-606004 missing event')\n", + "print('PASS: demo-606003 (malformed) has no event' if not bad_has_event else 'FAIL: demo-606003 unexpectedly has event')" + ] + }, + { + "cell_type": "markdown", + "id": "abb98930", + "metadata": {}, + "source": [ + "## Phase 7: live telescope-label resolution with fallback & failure reporting (TELESCOPE-02/03/04, SYNC-06/07/09)\n", + "\n", + "Phase 07 (v1.3) replaced the flat `parameters['site']` read with a per-record\n", + "live LCO Observation Portal API call (`_resolve_placement_block`), attempted\n", + "only for **placed** records (`scheduled_start`/`scheduled_end` populated,\n", + "D-01) and mapped through the verified `SITE_TELESCOPE_MAP`\n", + "`(site, aperture_class) -> 'SITECODE-CLASS'` dict. When that call fails,\n", + "times out, or returns a code absent from the dict, the record still gets a\n", + "`CalendarEvent` -- labeled with the coarse instrument-class fallback\n", + "(`1m0`/`0m4`/`2m0`/`4m0`), an `[UNVERIFIED]` title prefix, and a description\n", + "note -- instead of being skipped. A new `telescope_api_failed` counter,\n", + "distinct from `skipped`, tracks these fallbacks per facility.\n", + "\n", + "The cells below mock `make_request` (the same call site\n", + "`test_sync_lco_observation_calendar.py` patches) to demonstrate all three new\n", + "behaviors without making a real network call. Fixture records use the\n", + "`demo-607xxx` observation_id prefix, separate from the Phase-4/5/6 demos\n", + "above, and are removed by the teardown cell at the end of the notebook.\n" + ] + }, + { + "cell_type": "markdown", + "id": "d1d59ead", + "metadata": {}, + "source": [ + "### TELESCOPE-02: a placed record with a successful mocked API resolution gets a verified `SITECODE-CLASS` label and a clean title\n", + "\n", + "The record is placed (`scheduled_start`/`scheduled_end` set) at site `lsc`.\n", + "`make_request` is mocked to return a `COMPLETED` block for `site='lsc'`,\n", + "`telescope='1m0a'`, so `_resolve_placement_block` + `_derive_telescope`\n", + "resolve a verified `'LSC-1m0'` label -- no `[UNVERIFIED]` prefix, no fallback.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a99f40c1", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:56.544944Z", + "iopub.status.busy": "2026-06-25T05:47:56.544526Z", + "iopub.status.idle": "2026-06-25T05:47:56.703316Z", + "shell.execute_reply": "2026-06-25T05:47:56.700748Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ LCO from None to PENDING\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "title : LSC-1m0 1M0-SCICAM-SINISTRO\n", + "telescope : LSC-1m0\n", + "PASS: verified SITECODE-CLASS label, clean title\n" + ] + } + ], + "source": [ + "from datetime import datetime\n", + "from datetime import timezone as dt_timezone\n", + "from unittest.mock import MagicMock, patch\n", + "\n", + "ObservationRecord.objects.get_or_create(\n", + " observation_id='demo-607001',\n", + " defaults=dict(\n", + " target=demo_target,\n", + " user=demo_user,\n", + " facility='LCO',\n", + " status='PENDING',\n", + " scheduled_start=datetime(2026, 7, 20, 1, 0, 0, tzinfo=dt_timezone.utc),\n", + " scheduled_end=datetime(2026, 7, 20, 3, 0, 0, tzinfo=dt_timezone.utc),\n", + " parameters={\n", + " 'proposal': 'PHASE7-SUCCESS',\n", + " 'start': '2026-07-20T01:00:00',\n", + " 'end': '2026-07-20T03:00:00',\n", + " 'instrument_type': '1M0-SCICAM-SINISTRO',\n", + " 'site': 'lsc',\n", + " },\n", + " ),\n", + ")\n", + "\n", + "\n", + "def _observations_block_response(site, telescope, state='COMPLETED'):\n", + " \"\"\"Build a mock make_request() response for /api/requests/{id}/observations/.\"\"\"\n", + " response = MagicMock()\n", + " response.json.return_value = [{'site': site, 'enclosure': 'doma', 'telescope': telescope, 'state': state}]\n", + " return response\n", + "\n", + "\n", + "with patch(\n", + " 'solsys_code.management.commands.sync_lco_observation_calendar.make_request',\n", + " return_value=_observations_block_response(site='lsc', telescope='1m0a'),\n", + "):\n", + " success_stdout = io.StringIO()\n", + " call_command(\n", + " 'sync_lco_observation_calendar',\n", + " '--proposal',\n", + " 'PHASE7-SUCCESS',\n", + " stdout=success_stdout,\n", + " stderr=io.StringIO(),\n", + " )\n", + "\n", + "success_url = LCOFacility().get_observation_url('demo-607001')\n", + "event = CalendarEvent.objects.get(url=success_url)\n", + "print('title :', event.title)\n", + "print('telescope :', event.telescope)\n", + "\n", + "if event.telescope == 'LSC-1m0' and not event.title.startswith('[UNVERIFIED]'):\n", + " print('PASS: verified SITECODE-CLASS label, clean title')\n", + "else:\n", + " print('FAIL: expected LSC-1m0 + clean title, got', event.telescope, event.title)" + ] + }, + { + "cell_type": "markdown", + "id": "67815d00", + "metadata": {}, + "source": [ + "### TELESCOPE-03/04: a placed record with a mocked API failure falls back to a coarse label with an `[UNVERIFIED]` title prefix\n", + "\n", + "`make_request` is mocked to raise `requests.exceptions.Timeout` for this\n", + "record. Per TELESCOPE-03, the record is still synced (not skipped) with the\n", + "coarse instrument-class fallback label (`'1m0'`, derived from\n", + "`instrument_type='1M0-SCICAM-SINISTRO'`); per TELESCOPE-04, the title gets the\n", + "`[UNVERIFIED]` prefix and the description carries a generic\n", + "lookup-failed note (never the caught exception's content, per SYNC-09).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3afa36b1", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:56.707145Z", + "iopub.status.busy": "2026-06-25T05:47:56.706777Z", + "iopub.status.idle": "2026-06-25T05:47:56.822057Z", + "shell.execute_reply": "2026-06-25T05:47:56.813642Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ LCO from None to PENDING\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "title : [UNVERIFIED] 1m0 1M0-SCICAM-SINISTRO\n", + "telescope : 1m0\n", + "description : Proposal: PHASE7-FALLBACK\n", + "Status: PENDING\n", + "Window (UTC): 2026-07-21T01:00:00 to 2026-07-21T03:00:00\n", + "Telescope label unverified: live API lookup failed or returned an unmapped code.\n", + "stderr : Telescope API lookup failed or returned an unmapped code for observation_id='demo-607002'; using fallback label.\n", + "\n", + "PASS: [UNVERIFIED] prefix + coarse fallback label\n" + ] + } + ], + "source": [ + "import requests\n", + "\n", + "ObservationRecord.objects.get_or_create(\n", + " observation_id='demo-607002',\n", + " defaults=dict(\n", + " target=demo_target,\n", + " user=demo_user,\n", + " facility='LCO',\n", + " status='PENDING',\n", + " scheduled_start=datetime(2026, 7, 21, 1, 0, 0, tzinfo=dt_timezone.utc),\n", + " scheduled_end=datetime(2026, 7, 21, 3, 0, 0, tzinfo=dt_timezone.utc),\n", + " parameters={\n", + " 'proposal': 'PHASE7-FALLBACK',\n", + " 'start': '2026-07-21T01:00:00',\n", + " 'end': '2026-07-21T03:00:00',\n", + " 'instrument_type': '1M0-SCICAM-SINISTRO',\n", + " 'site': 'lsc',\n", + " },\n", + " ),\n", + ")\n", + "\n", + "fallback_stderr = io.StringIO()\n", + "with patch(\n", + " 'solsys_code.management.commands.sync_lco_observation_calendar.make_request',\n", + " side_effect=requests.exceptions.Timeout,\n", + "):\n", + " fallback_stdout = io.StringIO()\n", + " call_command(\n", + " 'sync_lco_observation_calendar',\n", + " '--proposal',\n", + " 'PHASE7-FALLBACK',\n", + " stdout=fallback_stdout,\n", + " stderr=fallback_stderr,\n", + " )\n", + "\n", + "fallback_url = LCOFacility().get_observation_url('demo-607002')\n", + "event = CalendarEvent.objects.get(url=fallback_url)\n", + "print('title :', event.title)\n", + "print('telescope :', event.telescope)\n", + "print('description :', event.description)\n", + "print('stderr :', fallback_stderr.getvalue())\n", + "\n", + "if event.title.startswith('[UNVERIFIED]') and event.telescope == '1m0':\n", + " print('PASS: [UNVERIFIED] prefix + coarse fallback label')\n", + "else:\n", + " print('FAIL: expected [UNVERIFIED]-prefixed title + 1m0 telescope, got', event.title, event.telescope)" + ] + }, + { + "cell_type": "markdown", + "id": "20f149b9", + "metadata": {}, + "source": [ + "### SYNC-06: the `telescope_api_failed` counter is distinct from `skipped` in the run summary\n", + "\n", + "The printed summary line from the fallback run above shows\n", + "`telescope_api_failed: 1` for the LCO facility, separate from\n", + "`skipped`/`extraction_failed` -- the record was never skipped, just\n", + "fallback-labeled.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "d5dbfab5", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:56.826438Z", + "iopub.status.busy": "2026-06-25T05:47:56.826028Z", + "iopub.status.idle": "2026-06-25T05:47:56.889789Z", + "shell.execute_reply": "2026-06-25T05:47:56.887623Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stdout: Done. proposal: PHASE7-FALLBACK, LCO: created: 1, updated: 0, unchanged: 0, skipped: 0, extraction_failed: 0, telescope_api_failed: 1 | SOAR: created: 0, updated: 0, unchanged: 0, skipped: 0, extraction_failed: 0, telescope_api_failed: 0\n", + "\n", + "PASS: telescope_api_failed counter visible and incremented in the summary\n" + ] + } + ], + "source": [ + "print('stdout:', fallback_stdout.getvalue())\n", + "\n", + "if 'telescope_api_failed: 1' in fallback_stdout.getvalue():\n", + " print('PASS: telescope_api_failed counter visible and incremented in the summary')\n", + "else:\n", + " print('FAIL: telescope_api_failed: 1 not found in summary output')" + ] + }, + { + "cell_type": "markdown", + "id": "24e702af", + "metadata": {}, + "source": [ + "### Phase 07.1: facility-aware coarse fallback label closes the SOAR doubled-title defect (TELESCOPE-03/04, SYNC-06)\n", + "\n", + "The v1.3 milestone audit found that `_coarse_telescope_label` only recognized LCO-style\n", + "aperture-class-prefixed `instrument_type` strings (e.g. `'1M0-SCICAM-SINISTRO'` ->\n", + "`'1m0'`). A SOAR instrument type such as `'SOAR_GHTS_REDCAM'` never matched that prefix\n", + "convention, so the raw instrument string fell through unchanged -- combined with the\n", + "`[UNVERIFIED] {telescope} {instrument}` title template, a placed SOAR record whose live\n", + "API call failed produced the doubled, non-coarse title\n", + "`'[UNVERIFIED] SOAR_GHTS_REDCAM SOAR_GHTS_REDCAM'` instead of a clean coarse label.\n", + "\n", + "`_coarse_telescope_label` is now facility-aware: it takes the record's `facility` string\n", + "(`record.facility`) as a second argument and returns `'4m0'` unconditionally for any SOAR\n", + "record -- SOAR has exactly one site and one aperture class per the single `('sor', '4m0')`\n", + "entry in `SITE_TELESCOPE_MAP`. The LCO branch is unchanged. The cell below mocks\n", + "`make_request` to fail for a placed SOAR record and shows the corrected `4m0` /\n", + "`[UNVERIFIED] 4m0 ...` label, not the doubled raw-instrument title.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "bb429918", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:56.896148Z", + "iopub.status.busy": "2026-06-25T05:47:56.895561Z", + "iopub.status.idle": "2026-06-25T05:47:57.005669Z", + "shell.execute_reply": "2026-06-25T05:47:57.002817Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Observation change state hook: sync-demo-target @ SOAR from None to PENDING\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "title : [UNVERIFIED] 4m0 SOAR_GHTS_REDCAM\n", + "telescope : 4m0\n", + "description : Proposal: PHASE71-SOAR-FALLBACK\n", + "Status: PENDING\n", + "Window (UTC): 2026-07-22T01:00:00 to 2026-07-22T03:00:00\n", + "Telescope label unverified: live API lookup failed or returned an unmapped code.\n", + "PASS: facility-aware 4m0 fallback label, clean [UNVERIFIED] 4m0 title, no doubled raw-instrument string\n" + ] + } + ], + "source": [ + "ObservationRecord.objects.get_or_create(\n", + " observation_id='demo-6071001',\n", + " defaults=dict(\n", + " target=demo_target,\n", + " user=demo_user,\n", + " facility='SOAR',\n", + " status='PENDING',\n", + " scheduled_start=datetime(2026, 7, 22, 1, 0, 0, tzinfo=dt_timezone.utc),\n", + " scheduled_end=datetime(2026, 7, 22, 3, 0, 0, tzinfo=dt_timezone.utc),\n", + " parameters={\n", + " 'proposal': 'PHASE71-SOAR-FALLBACK',\n", + " 'start': '2026-07-22T01:00:00',\n", + " 'end': '2026-07-22T03:00:00',\n", + " 'instrument_type': 'SOAR_GHTS_REDCAM',\n", + " 'site': 'sor',\n", + " },\n", + " ),\n", + ")\n", + "\n", + "soar_fallback_stderr = io.StringIO()\n", + "with patch(\n", + " 'solsys_code.management.commands.sync_lco_observation_calendar.make_request',\n", + " side_effect=requests.exceptions.Timeout,\n", + "):\n", + " soar_fallback_stdout = io.StringIO()\n", + " call_command(\n", + " 'sync_lco_observation_calendar',\n", + " '--proposal',\n", + " 'PHASE71-SOAR-FALLBACK',\n", + " stdout=soar_fallback_stdout,\n", + " stderr=soar_fallback_stderr,\n", + " )\n", + "\n", + "soar_fallback_url = LCOFacility().get_observation_url('demo-6071001')\n", + "event = CalendarEvent.objects.get(url=soar_fallback_url)\n", + "print('title :', event.title)\n", + "print('telescope :', event.telescope)\n", + "print('description :', event.description)\n", + "\n", + "if (\n", + " event.telescope == '4m0'\n", + " and event.title.startswith('[UNVERIFIED] 4m0 ')\n", + " and 'SOAR_GHTS_REDCAM SOAR_GHTS_REDCAM' not in event.title\n", + "):\n", + " print('PASS: facility-aware 4m0 fallback label, clean [UNVERIFIED] 4m0 title, no doubled raw-instrument string')\n", + "else:\n", + " print('FAIL: expected 4m0 telescope + [UNVERIFIED] 4m0 title, got', event.telescope, event.title)" + ] + }, + { + "cell_type": "markdown", + "id": "5f6ca417", + "metadata": {}, + "source": [ + "## Phase 8: telescope-label verification sidecar (DISPLAY-01)\n", + "\n", + "Phase 8 turns the existing transient `telescope_api_failed` signal (Phase 07/07.1) into a\n", + "queryable, structured fact: a `CalendarEventTelescopeLabel` sidecar row\n", + "(`OneToOneField(primary_key=True)` on `CalendarEvent`) recording whether the event's\n", + "telescope label was live-verified against the LCO API or fallback-guessed. The sync\n", + "command writes this row via `update_or_create`, immediately after creating or updating\n", + "the `CalendarEvent` itself, for every record it processes.\n" + ] + }, + { + "cell_type": "markdown", + "id": "c11e6e92", + "metadata": {}, + "source": [ + "### A verified record gets a sidecar row with `is_verified=True`\n", + "\n", + "Re-using the placed LCO fixture from the Phase 07 cells above (`make_request` mocked to\n", + "return a successful, mappable site/telescope block): after the sync run, the created\n", + "`CalendarEvent` has an associated `CalendarEventTelescopeLabel` row, and that row's\n", + "`is_verified` is `True`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "1b5ac1e1", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:57.010412Z", + "iopub.status.busy": "2026-06-25T05:47:57.009933Z", + "iopub.status.idle": "2026-06-25T05:47:57.091728Z", + "shell.execute_reply": "2026-06-25T05:47:57.089749Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "event title : LSC-1m0 1M0-SCICAM-SINISTRO\n", + "is_verified : True\n", + "Verified label for LSC-1m0 1M0-SCICAM-SINISTRO\n", + "PASS: verified record has a sidecar row with is_verified=True\n" + ] + } + ], + "source": [ + "from solsys_code.models import CalendarEventTelescopeLabel\n", + "\n", + "verified_url = LCOFacility().get_observation_url('demo-607001')\n", + "verified_event = CalendarEvent.objects.get(url=verified_url)\n", + "verified_label = CalendarEventTelescopeLabel.objects.get(event=verified_event)\n", + "\n", + "print('event title :', verified_event.title)\n", + "print('is_verified :', verified_label.is_verified)\n", + "print(verified_label)\n", + "\n", + "if verified_label.is_verified is True:\n", + " print('PASS: verified record has a sidecar row with is_verified=True')\n", + "else:\n", + " print('FAIL: expected is_verified=True for a successfully-verified record')" + ] + }, + { + "cell_type": "markdown", + "id": "ded6518a", + "metadata": {}, + "source": [ + "### A fallback-labeled record gets a sidecar row with `is_verified=False`\n", + "\n", + "Re-using the fallback LCO fixture from the Phase 07 cells above (`make_request` mocked\n", + "to time out): the sidecar row created for that same record has `is_verified=False`,\n", + "matching the `[UNVERIFIED]`-prefixed title and coarse fallback telescope label already\n", + "demonstrated.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "738d203d", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:57.097636Z", + "iopub.status.busy": "2026-06-25T05:47:57.097135Z", + "iopub.status.idle": "2026-06-25T05:47:57.174322Z", + "shell.execute_reply": "2026-06-25T05:47:57.167976Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "event title : [UNVERIFIED] 1m0 1M0-SCICAM-SINISTRO\n", + "is_verified : False\n", + "Fallback label for [UNVERIFIED] 1m0 1M0-SCICAM-SINISTRO\n", + "PASS: fallback-labeled record has a sidecar row with is_verified=False\n" + ] + } + ], + "source": [ + "fallback_url = LCOFacility().get_observation_url('demo-607002')\n", + "fallback_event = CalendarEvent.objects.get(url=fallback_url)\n", + "fallback_label = CalendarEventTelescopeLabel.objects.get(event=fallback_event)\n", + "\n", + "print('event title :', fallback_event.title)\n", + "print('is_verified :', fallback_label.is_verified)\n", + "print(fallback_label)\n", + "\n", + "if fallback_label.is_verified is False:\n", + " print('PASS: fallback-labeled record has a sidecar row with is_verified=False')\n", + "else:\n", + " print('FAIL: expected is_verified=False for a fallback-labeled record')" + ] + }, + { + "cell_type": "markdown", + "id": "c86af044", + "metadata": {}, + "source": [ + "### A classically-scheduled event (load_telescope_runs) has no sidecar row at all\n", + "\n", + "`load_telescope_runs` never calls the LCO API and has no telescope-label-resolution\n", + "concept, so it never writes a `CalendarEventTelescopeLabel` row. A missing row means\n", + "\"verified\" by documented default. We demonstrate this by creating one classically-\n", + "scheduled event directly (mirroring what `load_telescope_runs` produces) and showing\n", + "that accessing its `telescope_label_meta` reverse accessor raises `DoesNotExist`,\n", + "rather than fabricating a row for an event the sidecar's only writer never touches.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "6c6badfc", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:57.185378Z", + "iopub.status.busy": "2026-06-25T05:47:57.184838Z", + "iopub.status.idle": "2026-06-25T05:47:57.264638Z", + "shell.execute_reply": "2026-06-25T05:47:57.262601Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PASS: classically-scheduled event has no CalendarEventTelescopeLabel row (reverse accessor raises DoesNotExist, meaning \"verified\" by default)\n" + ] + } + ], + "source": [ + "from tom_calendar.models import CalendarEvent as _CalendarEvent\n", + "\n", + "classical_event = _CalendarEvent.objects.create(\n", + " title='NTT EFOSC2',\n", + " description='Demo classically-scheduled event (Phase 8 no-sidecar-row demo)',\n", + " telescope='NTT',\n", + " instrument='EFOSC2',\n", + " start_time=datetime(2026, 7, 23, 0, 0, 0, tzinfo=dt_timezone.utc),\n", + " end_time=datetime(2026, 7, 23, 10, 0, 0, tzinfo=dt_timezone.utc),\n", + ")\n", + "\n", + "try:\n", + " _ = classical_event.telescope_label_meta\n", + " print('FAIL: expected CalendarEventTelescopeLabel.DoesNotExist, but a row exists')\n", + "except CalendarEventTelescopeLabel.DoesNotExist:\n", + " print(\n", + " 'PASS: classically-scheduled event has no CalendarEventTelescopeLabel row '\n", + " '(reverse accessor raises DoesNotExist, meaning \"verified\" by default)'\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "bdd4a485", + "metadata": {}, + "source": [ + "### Phase 8 summary\n", + "\n", + "This confirms DISPLAY-01: every synced `CalendarEvent` gets a `CalendarEventTelescopeLabel`\n", + "sidecar row whose `is_verified` matches `not telescope_api_failed` for that record, while\n", + "events created by `load_telescope_runs` (classical schedule, no API call) have no sidecar\n", + "row at all and read as verified by documented default.\n" + ] + }, + { + "cell_type": "markdown", + "id": "b4c5d6e4", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "This notebook demonstrates the Phase 04 `sync_lco_observation_calendar` command\n", + "satisfying requirements SELECT-01, SYNC-01, SYNC-02, SYNC-03, SYNC-04, and TERM-01,\n", + "Phase 05's SELECT-02/03/04/05 (multi-proposal and multi-facility selection),\n", + "Phase 06's EXTRACT-01/EXTRACT-02 (correct instrument-type extraction), and\n", + "Phase 07's TELESCOPE-02/03/04 and SYNC-06/07/09 (live telescope-label resolution\n", + "with fallback & failure reporting): Phase 07.1 closes a v1.3 milestone-audit gap in that same TELESCOPE-03/TELESCOPE-04/SYNC-06 fallback path: the coarse label is now facility-aware so a SOAR record's fallback label is `'4m0'`, not the raw, doubled instrument string.\n", + "\n", + "| Requirement | Description | Demonstrated by |\n", + "|-------------|-------------|------------------|\n", + "| SELECT-01 | Only `ObservationRecord`s matching the given `--proposal` are synced | `--proposal DEMOCODE` selects the fixture record |\n", + "| SYNC-01 | Event `url` keyed on `LCOFacility().get_observation_url(observation_id)`, containing `/requests/` | url printed and used as the lookup key throughout |\n", + "| SYNC-02 | Unscheduled record (`scheduled_start=None`) uses `parameters['start']`/`['end']` and gets a `[QUEUED]` title prefix | first sync, \"Inspect the queued CalendarEvent\" |\n", + "| SYNC-03 | Scheduled record (`scheduled_start`/`scheduled_end` populated) uses those times and a clean title (no `[QUEUED]`) | \"Simulate the LCO scheduler placing the observation\" |\n", + "| SYNC-04 | Re-running with no changes leaves `modified` untouched and reports `unchanged: 1` | \"No-churn idempotency\" |\n", + "| TERM-01 | Terminal failure statuses get a `[EXPIRED]`/`[CANCELLED]`/`[FAILED]` title prefix | not exercised by this fixture; see `solsys_code/tests/test_sync_lco_observation_calendar.py` for the terminal-state test cases |\n", + "| SELECT-02 | comma-list `--proposal` matches exactly the listed codes with no substring match on a decoy | the SELECT-02 cells (decoy `PHASE5-AB` gets no event) |\n", + "| SELECT-03 | `--proposal all` (any casing) syncs every record regardless of proposal | the SELECT-03 cells (lowercase `all` syncs everything) |\n", + "| SELECT-04 | a single run produces CalendarEvents for both an LCO and a SOAR record | the SELECT-04 cells (one invocation, both facilities) |\n", + "| SELECT-05 | SOAR records dispatched through a SOARFacility instance, never a reused LCOFacility | not demonstrated here (identical url strings); verified by `test_select_05_soar_record_uses_soar_facility_instance` |\n", + "| EXTRACT-01 | the command no longer reads a flat instrument_type blindly when c_N_* multi-config keys are present | the Phase 6 cells below (SOAR/MUSCAT fixtures both carry a decoy flat instrument_type that is correctly ignored) |\n", + "| EXTRACT-02 | the science configuration (SPECTRUM/EXPOSE/etc.) is picked over SOAR calibration configs (ARC/LAMP_FLAT) or via MUSCAT's per-channel exposure signal when no config has a recognized configuration_type; a record with neither signal is skipped and counted under a dedicated extraction_failed counter | the SOAR multi-config cell, the MUSCAT per-channel cell, and the malformed-record cell (extraction_failed: 1 in the printed summary) |\n", + "| TELESCOPE-02 | a placed record resolves its telescope label via a live API call, mapped through the verified `SITE_TELESCOPE_MAP` | the TELESCOPE-02 success cell (`'LSC-1m0'`, clean title) |\n", + "| TELESCOPE-03 | an API call failure/timeout/unmapped-code for a placed record falls back to the coarse instrument-class label instead of being skipped | the TELESCOPE-03/04 fallback cell (`'1m0'` telescope, event still created) |\n", + "| TELESCOPE-04 | a fallback-labeled event is visibly distinguishable: `[UNVERIFIED]` title prefix, coarse telescope token, description failure note | the TELESCOPE-03/04 fallback cell |\n", + "| SYNC-06 | a dedicated `telescope_api_failed` counter, distinct from `skipped`/`extraction_failed`, reported per facility in the summary | the SYNC-06 cell (`telescope_api_failed: 1` in the printed summary) |\n", + "| SYNC-07 | a per-record API failure never aborts the run | the fallback cell completes via `call_command` with no raised exception |\n", + "| SYNC-09 | the failure log line and fallback description never contain the response body or API key | the fallback cell's printed stderr (generic message + observation_id only) |\n", + "| TELESCOPE-03 (SOAR gap fix) | a placed SOAR record's API-failure fallback label is facility-aware: `_coarse_telescope_label` returns `'4m0'` unconditionally for SOAR instead of the raw, unmatched instrument string | the Phase 07.1 SOAR-fallback cell (`'4m0'` telescope, clean `[UNVERIFIED] 4m0` title) |\n", + "| TELESCOPE-04 (SOAR gap fix) | the previously doubled `'[UNVERIFIED] SOAR_GHTS_REDCAM SOAR_GHTS_REDCAM'` title no longer occurs for a SOAR fallback | the Phase 07.1 SOAR-fallback cell (asserts the doubled string is absent) |\n", + "| SYNC-06 (SOAR gap fix) | the SOAR fallback record is counted as a degrade (not skipped), closing the v1.3 milestone audit's zero-coverage gap for SOAR+placed+API-failure | the Phase 07.1 SOAR-fallback cell and `test_telescope_03_soar_api_failure_fallback_returns_4m0_label` |\n", + "\n", + "The command is invoked via `call_command` — equivalent to running\n", + "`./manage.py sync_lco_observation_calendar --proposal ` from the shell.\n", + "\n", + "This notebook is **pre-executed** and intentionally excluded from automated doc\n", + "builds (not referenced in `docs/notebooks.rst`) because it depends on a live\n", + "`ObservationRecord`/`CalendarEvent` fixture in the local dev DB.\n", + "Phase 8 (v1.4) adds DISPLAY-01: a `CalendarEventTelescopeLabel` sidecar model persisting whether each synced event's telescope label was live-verified or fallback-guessed, written via a standalone `update_or_create` call colocated with the existing `CalendarEvent` write -- with no row at all for classically-scheduled (`load_telescope_runs`) events.\n" + ] + }, + { + "cell_type": "markdown", + "id": "c5d6e7f5", + "metadata": {}, + "source": [ + "## Teardown\n", + "\n", + "This notebook is DB-dependent and creates real rows in the local dev\n", + "database. Clean up everything created above so re-running the notebook\n", + "(or other notebooks/tests sharing the dev DB) starts from a clean slate:\n", + "the `CalendarEvent`s (Phase-4 and Phase-5), the `ObservationRecord`s\n", + "(Phase-4 and Phase-5), the demo `Target` (whose factory-generated\n", + "`TargetExtra`/`TargetName` rows cascade-delete with it), and the demo\n", + "`User`.\n", + "\n", + "Deletion order matters: `ObservationRecord.user` is `on_delete=DO_NOTHING`,\n", + "so the records must be deleted before the user. `CalendarEvent` has no FK\n", + "to either `ObservationRecord` or `Target` (it's matched by `url` in the\n", + "sync command), so it must be deleted explicitly rather than relying on\n", + "cascade.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "d6e7f8a6", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-25T05:47:57.270842Z", + "iopub.status.busy": "2026-06-25T05:47:57.270240Z", + "iopub.status.idle": "2026-06-25T05:47:57.902295Z", + "shell.execute_reply": "2026-06-25T05:47:57.899770Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Total removed orphan object permissions instances: 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Teardown complete: demo CalendarEvents, ObservationRecords (Phase-4, Phase-5, Phase-6, Phase-7, Phase-07.1, and Phase-8), Target, and User removed.\n" + ] + } + ], + "source": [ + "# Phase-5 fixtures created in the cells above.\n", + "demo_phase5_all_ids = demo_phase5_select02_ids + [\n", + " 'demo-603001',\n", + " 'demo-603002',\n", + " 'demo-604001',\n", + " 'demo-604002',\n", + "]\n", + "\n", + "for observation_id in demo_phase5_all_ids:\n", + " url = LCOFacility().get_observation_url(observation_id)\n", + " CalendarEvent.objects.filter(url=url).delete()\n", + " ObservationRecord.objects.filter(observation_id=observation_id).delete()\n", + "\n", + "# Phase-6 fixtures created in the cells above.\n", + "demo_phase6_all_ids = ['demo-606001', 'demo-606002', 'demo-606003', 'demo-606004']\n", + "\n", + "for observation_id in demo_phase6_all_ids:\n", + " url = LCOFacility().get_observation_url(observation_id)\n", + " CalendarEvent.objects.filter(url=url).delete()\n", + " ObservationRecord.objects.filter(observation_id=observation_id).delete()\n", + "\n", + "# Phase-7 fixtures created in the cells above.\n", + "demo_phase7_all_ids = ['demo-607001', 'demo-607002']\n", + "\n", + "for observation_id in demo_phase7_all_ids:\n", + " url = LCOFacility().get_observation_url(observation_id)\n", + " CalendarEvent.objects.filter(url=url).delete()\n", + " ObservationRecord.objects.filter(observation_id=observation_id).delete()\n", + "\n", + "# Phase-07.1 fixture cleanup (SOAR facility-aware fallback label demo).\n", + "demo_phase71_all_ids = ['demo-6071001']\n", + "\n", + "for observation_id in demo_phase71_all_ids:\n", + " url = LCOFacility().get_observation_url(observation_id)\n", + " CalendarEvent.objects.filter(url=url).delete()\n", + " ObservationRecord.objects.filter(observation_id=observation_id).delete()\n", + "\n", + "# Phase-8 fixture cleanup (telescope-label verification sidecar demo).\n", + "_CalendarEvent.objects.filter(pk=classical_event.pk).delete()\n", + "\n", + "# Phase-4 fixture cleanup.\n", + "CalendarEvent.objects.filter(url=event_url).delete()\n", + "ObservationRecord.objects.filter(observation_id='demo-900001').delete()\n", + "\n", + "# Shared Target/User cleanup (reused by both Phase-4 and Phase-5 fixtures above).\n", + "Target.objects.filter(name=DEMO_TARGET_NAME).delete()\n", + "get_user_model().objects.filter(username='sync-demo-user').delete()\n", + "\n", + "print(\n", + " 'Teardown complete: demo CalendarEvents, ObservationRecords '\n", + " '(Phase-4, Phase-5, Phase-6, Phase-7, Phase-07.1, and Phase-8), '\n", + " 'Target, and User removed.'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "b1580ddf", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "fomo311_venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/notebooks/pre_executed/telescope_runs_demo.ipynb b/docs/notebooks/pre_executed/telescope_runs_demo.ipynb new file mode 100644 index 00000000..3d8c582e --- /dev/null +++ b/docs/notebooks/pre_executed/telescope_runs_demo.ipynb @@ -0,0 +1,371 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c87b5719", + "metadata": {}, + "source": [ + "# Telescope Runs — Stage 1 Demo (site / sun-event helper)\n", + "\n", + "This notebook demonstrates `solsys_code/telescope_runs.py` (issue #37 Stage 1), the\n", + "site/ephemeris helper that resolves a telescope name to its `Observatory` record\n", + "and computes dip-corrected UTC sun-event times for a given date.\n", + "\n", + "It exercises three helpers:\n", + "\n", + "- `get_site(name)` — resolves a telescope name (a key of `SITES`) to an\n", + " `Observatory` model record.\n", + "- `horizon_dip(altitude_m)` — computes the horizon dip correction (in degrees)\n", + " for an observer at the given altitude.\n", + "- `sun_event(site, date, kind)` — returns (setting, rising) UTC `Time` objects\n", + " for either the dip-corrected sun event (`kind='sun'`) or the -15°\n", + " astronomical-dark window (`kind='dark'`).\n", + "\n", + "This notebook lives in `pre_executed/` because it is **DB-dependent** (it needs\n", + "`Observatory` records to exist) and is therefore **NOT run during\n", + "Sphinx/CI/ReadTheDocs builds**, per `docs/notebooks/README.md`'s guidance to put\n", + "notebooks that need resources not guaranteed in every environment under\n", + "`pre_executed/`." + ] + }, + { + "cell_type": "markdown", + "id": "23326d63", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "\n", + "This notebook requires `Observatory` records for MPC obscodes `'268'`\n", + "(Magellan-Clay), `'269'` (Magellan-Baade), `'809'` (NTT / La Silla), and\n", + "`'E10'` (FTS / Siding Spring) to exist in the database. These can be created\n", + "via the `CreateObservatory` form (or an equivalent fixture/migration).\n", + "\n", + "The site coordinates and timezones used below are documented in\n", + "`solsys_code/tests/test_telescope_runs.py`'s `setUpTestData`:\n", + "\n", + "| obscode | short_name | lat | lon | altitude (m) | timezone |\n", + "|---|---|---|---|---|---|\n", + "| 268 | Magellan-Clay | -29.0146 | -70.6926 | 2402 | America/Santiago |\n", + "| 269 | Magellan-Baade | -29.0146 | -70.6926 | 2402 | America/Santiago |\n", + "| 809 | NTT | -29.2567 | -70.7300 | 2347 | America/Santiago |\n", + "| E10 | FTS | -31.2734 | 149.0612 | 1149 | Australia/Sydney |" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "bc94a65e", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-01T14:27:13.041151Z", + "iopub.status.busy": "2026-07-01T14:27:13.040540Z", + "iopub.status.idle": "2026-07-01T14:27:18.491495Z", + "shell.execute_reply": "2026-07-01T14:27:18.488810Z" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "import django\n", + "\n", + "# Ensure the repo root is on sys.path so `src.fomo.settings` is importable\n", + "# when this notebook is executed from docs/notebooks/pre_executed/.\n", + "repo_root = str(Path.cwd().resolve().parents[2])\n", + "if repo_root not in sys.path:\n", + " sys.path.insert(0, repo_root)\n", + "\n", + "os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'src.fomo.settings')\n", + "\n", + "# Jupyter's ipykernel runs inside an asyncio event loop, but Django's ORM is\n", + "# sync-only by default and refuses to run there; this opts back in.\n", + "os.environ.setdefault('DJANGO_ALLOW_ASYNC_UNSAFE', 'true')\n", + "\n", + "django.setup()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "82ddad64", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-01T14:27:18.505096Z", + "iopub.status.busy": "2026-07-01T14:27:18.504287Z", + "iopub.status.idle": "2026-07-01T14:27:18.652285Z", + "shell.execute_reply": "2026-07-01T14:27:18.649397Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'Magellan-Clay': '268', 'Magellan-Baade': '269', 'NTT': '809', 'FTS': 'E10'}\n" + ] + } + ], + "source": [ + "from datetime import date\n", + "\n", + "from solsys_code.telescope_runs import SITES, get_site, horizon_dip, sun_event\n", + "\n", + "print(SITES)" + ] + }, + { + "cell_type": "markdown", + "id": "b6e2ef71", + "metadata": {}, + "source": [ + "## Resolving a telescope to its site\n", + "\n", + "`get_site(name)` looks up `name` in `SITES` to get an MPC obscode, then returns\n", + "the corresponding `Observatory` record. This is the single source of truth for\n", + "the site's geodetic position (lat/lon/altitude) and IANA timezone." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "21514e41", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-01T14:27:18.672350Z", + "iopub.status.busy": "2026-07-01T14:27:18.669679Z", + "iopub.status.idle": "2026-07-01T14:27:18.743012Z", + "shell.execute_reply": "2026-07-01T14:27:18.740604Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "short_name: European Southern Observatory, La Silla\n", + "obscode: 809\n", + "lat: -29.25881957385624\n", + "lon: -70.73374000000003\n", + "altitude: 2345.365187853709\n", + "timezone: America/Santiago\n" + ] + } + ], + "source": [ + "site = get_site('NTT')\n", + "\n", + "print('short_name:', site.short_name)\n", + "print('obscode:', site.obscode)\n", + "print('lat:', site.lat)\n", + "print('lon:', site.lon)\n", + "print('altitude:', site.altitude)\n", + "print('timezone:', site.timezone)" + ] + }, + { + "cell_type": "markdown", + "id": "35f70d21", + "metadata": {}, + "source": [ + "## Horizon dip\n", + "\n", + "An observer above sea level sees the true horizon dip below the astronomical\n", + "horizon. The correction is:\n", + "\n", + "```\n", + "dip = 1.76' * sqrt(altitude_m) (in arcminutes, converted to degrees)\n", + "```\n", + "\n", + "This is added to the standard refraction + solar semi-diameter offset\n", + "(0.833°) when computing sunset/sunrise, so higher sites see the Sun set\n", + "slightly later (and rise slightly earlier) than sea-level observers." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d3f7422c", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-01T14:27:18.757298Z", + "iopub.status.busy": "2026-07-01T14:27:18.754086Z", + "iopub.status.idle": "2026-07-01T14:27:18.817950Z", + "shell.execute_reply": "2026-07-01T14:27:18.812669Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "horizon dip at 2345 m: 85.2350 arcmin deg\n" + ] + } + ], + "source": [ + "dip = horizon_dip(site.altitude)\n", + "print(f'horizon dip at {site.altitude:.0f} m: {dip:.4f} deg')\n", + "\n", + "# Reference (EPHEM-03): at Las Campanas (2402 m), dip should be 1.44 deg +/- 0.02." + ] + }, + { + "cell_type": "markdown", + "id": "025330d9", + "metadata": {}, + "source": [ + "## Sun events\n", + "\n", + "`sun_event(site, date, kind)` returns `(setting, rising)` as UTC `astropy.time.Time`\n", + "objects for the observing night that starts on the evening of `date`:\n", + "\n", + "- `kind='sun'` — dip-corrected sunset/sunrise (solar altitude crosses\n", + " `-(0.833 + dip)` degrees).\n", + "- `kind='dark'` — astronomical-dark window (solar altitude crosses -15\n", + " degrees, no dip correction)." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "e741196f", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-01T14:27:18.829797Z", + "iopub.status.busy": "2026-07-01T14:27:18.829167Z", + "iopub.status.idle": "2026-07-01T14:27:21.374822Z", + "shell.execute_reply": "2026-07-01T14:27:21.372408Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sunset : 2026-06-10 21:58:45.352\n", + "sunrise : 2026-06-11 11:26:24.434\n", + "dark start: 2026-06-10 23:01:43.008\n", + "dark end : 2026-06-11 10:23:24.844\n" + ] + } + ], + "source": [ + "sample_date = date(2026, 6, 10)\n", + "\n", + "sunset, sunrise = sun_event(site, sample_date, 'sun')\n", + "dark_start, dark_end = sun_event(site, sample_date, 'dark')\n", + "\n", + "print('sunset :', sunset.iso)\n", + "print('sunrise :', sunrise.iso)\n", + "print('dark start:', dark_start.iso)\n", + "print('dark end :', dark_end.iso)" + ] + }, + { + "cell_type": "markdown", + "id": "e4ebf7a3", + "metadata": {}, + "source": [ + "## Parsing run lines with optional partial-night window tokens\n", + "\n", + "`parse_run_line` accepts an optional trailing `(BoN|HHMM)-(EoN|HHMM)` token that\n", + "restricts the observing window to a portion of the night. The token is stored in\n", + "the `start_window` and `end_window` fields of the returned `ParsedRun`.\n", + "\n", + "- `BoN` — beginning of night (computed sunset)\n", + "- `EoN` — end of night (computed sunrise)\n", + "- `HHMM` — a fixed UTC time: if HHMM < 1200 it is treated as next-morning UTC\n", + " (evening_date + 1 day); if HHMM ≥ 1200 it is evening-date UTC.\n", + "\n", + "When `start_window` / `end_window` are `None` the full night (sunset → sunrise) is used." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cab73c00", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-01T14:27:21.379573Z", + "iopub.status.busy": "2026-07-01T14:27:21.379016Z", + "iopub.status.idle": "2026-07-01T14:27:21.413960Z", + "shell.execute_reply": "2026-07-01T14:27:21.412737Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Full night:\n", + " start_window=None end_window=None\n", + "\n", + "First-half night (BoN-0626):\n", + " start_window='BoN' end_window='0626'\n", + "\n", + "Second-half night (0646-EoN):\n", + " start_window='0646' end_window='EoN'\n" + ] + } + ], + "source": [ + "from solsys_code.telescope_runs import parse_run_line\n", + "\n", + "# Full night — start_window and end_window default to None\n", + "full = parse_run_line('NTT EFOSC2 allocation 9-13 July')\n", + "print('Full night:')\n", + "print(f' start_window={full.start_window!r} end_window={full.end_window!r}')\n", + "\n", + "# First half of night — BoN (sunset) to 06:26 UTC next morning\n", + "first_half = parse_run_line('Magellan-Clay Lightspeed 18-20 July BoN-0626')\n", + "print('\\nFirst-half night (BoN-0626):')\n", + "print(f' start_window={first_half.start_window!r} end_window={first_half.end_window!r}')\n", + "\n", + "# Second half of night — 06:46 UTC next morning to EoN (sunrise)\n", + "second_half = parse_run_line('Magellan-Clay LDSS3 18-20 July 0646-EoN')\n", + "print('\\nSecond-half night (0646-EoN):')\n", + "print(f' start_window={second_half.start_window!r} end_window={second_half.end_window!r}')" + ] + }, + { + "cell_type": "markdown", + "id": "88d0bc33", + "metadata": {}, + "source": [ + "## Closing notes\n", + "\n", + "For Las Campanas (Magellan-Clay / Magellan-Baade, obscodes 268/269), the computed\n", + "sun-event times for June 2026 match the LCO skycalc reference tool to within 2\n", + "minutes (EPHEM-04). NTT (La Silla, obscode 809) is part of the same site cluster\n", + "and uses the same `America/Santiago` timezone.\n", + "\n", + "This notebook is **pre-executed** and intentionally excluded from automated doc\n", + "builds (it is not referenced in `docs/notebooks.rst`) because it depends on\n", + "`Observatory` records existing in the database." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/manage.py b/manage.py index 637e906d..817eeb0e 100644 --- a/manage.py +++ b/manage.py @@ -1,5 +1,6 @@ #!/usr/bin/env python """Django's command-line utility for administrative tasks.""" + import os import sys diff --git a/pyproject.toml b/pyproject.toml index 09fe1409..5990a181 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ classifiers = [ dynamic = ["version"] requires-python = ">=3.10" dependencies = [ - "tomtoolkit>=3.0.0a9", # pre-release specifier opts into the 3.0 alpha required by tom_jpl/tom_fink + "tomtoolkit==3.0.0a9", # pre-release specifier opts into the 3.0 alpha required by tom_jpl/tom_fink "tom_alertstreams>=1.2.1", "tom_fink>=1.0.0", "tom-registration>=1.0.5", @@ -25,6 +25,7 @@ dependencies = [ "numpy>1.24", "sbpy>=0.6.0", # We don't use sbpy directly, but sorcha does and sbpy 0.6.0 is needed for astropy 7.2.0+ "sorcha", + "timezonefinder>=6.0", ] [project.urls] @@ -70,7 +71,7 @@ addopts = "--doctest-modules --doctest-glob=*.rst" [tool.ruff] line-length = 120 target-version = "py310" -exclude = ["solsys_code/**/migrations/*.py", ] +exclude = ["solsys_code/**/migrations/*.py", "docs/notebooks/ESO_How_to_download_data.ipynb"] [tool.ruff.format] quote-style = "single" diff --git a/solsys_code/admin.py b/solsys_code/admin.py index 4185d360..c5cfa9fd 100644 --- a/solsys_code/admin.py +++ b/solsys_code/admin.py @@ -1,3 +1,34 @@ -# from django.contrib import admin +from django.contrib import admin -# Register your models here. +from solsys_code.models import CalendarEventTelescopeLabel, CampaignRun + + +class CampaignRunAdmin(admin.ModelAdmin): # noqa: D101 + list_display = [ + 'pk', + 'campaign', + 'telescope_instrument', + 'approval_status', + 'run_status', + 'site', + 'window_start', + 'window_end', + ] + list_filter = ['approval_status', 'run_status', 'campaign'] + search_fields = ['telescope_instrument', 'site_raw', 'contact_person'] + # approval_status must stay read-only here: its normal transition triggers the + # calendar-projection side effect and the D-06 `if run.site is None` clobber guard that + # live entirely in CampaignRunDecisionView.post(), not on the model. Admin must never be + # able to silently flip a run to APPROVED without going through that real approval-queue + # flow. + readonly_fields = ['approval_status'] + + +class CalendarEventTelescopeLabelAdmin(admin.ModelAdmin): # noqa: D101 + list_display = ['event', 'is_verified'] + list_filter = ['is_verified'] + search_fields = ['event__title'] + + +admin.site.register(CampaignRun, CampaignRunAdmin) +admin.site.register(CalendarEventTelescopeLabel, CalendarEventTelescopeLabelAdmin) diff --git a/solsys_code/apps.py b/solsys_code/apps.py index 90b58d24..c696c04d 100644 --- a/solsys_code/apps.py +++ b/solsys_code/apps.py @@ -13,6 +13,22 @@ def target_detail_buttons(self): { 'partial': f'{self.name}/partials/ephem_button.html', 'context': 'src.templatetags.solsys_code_extras.ephem_button', + }, + { + 'partial': f'{self.name}/partials/campaign_links.html', + 'context': 'src.templatetags.solsys_code_extras.campaign_links', + }, + ] + + def nav_items(self): + """ + Integration point for adding entries to the navbar (VIEW-02/D-03). + """ + return [ + { + 'partial': f'{self.name}/partials/campaigns_nav_link.html', + 'context': 'src.templatetags.solsys_code_extras.campaigns_nav_link', + 'position': 'left', } ] diff --git a/solsys_code/calendar_urls.py b/solsys_code/calendar_urls.py new file mode 100644 index 00000000..c401facd --- /dev/null +++ b/solsys_code/calendar_urls.py @@ -0,0 +1,23 @@ +"""FOMO-local calendar URL conf — full replacement of tom_calendar.urls for /calendar/. + +Shadows the entire tom_calendar URL namespace so that all calendar:* reversals resolve +through this module. The root path ('') is served by fomo_render_calendar, which injects +prefetch_related + Count annotation (DISPLAY-09). All sub-paths (create, update, delete, +todo) delegate to the upstream tom_calendar view functions unchanged. +""" + +from django.urls import path +from tom_calendar.views import create_event, create_todo, delete_event, update_event, update_todo + +from solsys_code.views import fomo_render_calendar + +app_name = 'calendar' + +urlpatterns = [ + path('', fomo_render_calendar, name='calendar'), + path('create/', create_event, name='create-event'), + path('update//', update_event, name='update-event'), + path('delete//', delete_event, name='delete-event'), + path('todo/create//', create_todo, name='create-todo'), + path('todo/update//', update_todo, name='update-todo'), +] diff --git a/solsys_code/calendar_utils.py b/solsys_code/calendar_utils.py new file mode 100644 index 00000000..ee7093e7 --- /dev/null +++ b/solsys_code/calendar_utils.py @@ -0,0 +1,378 @@ +"""Shared LCO/SOAR telescope-mapping helpers and CalendarEvent create-or-update helper. + +Provides the instrument-extraction chain and telescope-mapping constants extracted from +sync_lco_observation_calendar so all three management commands (sync_lco, +sync_gemini, load_telescope_runs) can share a single implementation, plus the +no-churn CalendarEvent create-or-update function used by all three consumers. +""" + +from datetime import timedelta +from typing import Any +from urllib.parse import urljoin + +import requests +from django import forms +from tom_calendar.models import CalendarEvent +from tom_common.exceptions import ImproperCredentialsException +from tom_observations.facilities.lco import LCOFacility +from tom_observations.facilities.ocs import make_request + +# (site, aperture_class) -> 'SITECODE-CLASS' telescope label (TELESCOPE-01/D-03/D-04). +# Verified, real-data-grounded inventory of the 7 real LCO-network sites this +# codebase's installed LCOSettings/SOARSettings actually confirm (tlv/Wise Observatory +# is deliberately excluded -- confirmed absent from both installed get_sites() +# implementations at the 07-01 Task 1 checkpoint; see 07-01-SUMMARY.md Deviations). +# 'coj'/'ogg'/'sor' migrate the 3 pre-existing entries (D-05) -- 'coj'/'ogg' confirmed +# 2m0 (FTS/FTN), 'sor' confirmed 4m0 (SOAR, tom_observations.facilities.soar hardcodes +# 'sitecode': 'sor'). 'elp'/'lsc'/'cpt'/'tfn' confirmed by operator (LCO staff) at the +# 07-01 Task 1 checkpoint -- see 07-01-SUMMARY.md -- as standard 1m-network sites +# hosting both 1m0 and 0m4 telescope classes. 'coj' (Siding Spring) and 'ogg' (Haleakala) +# additionally host 0m4/1m0 (coj) and 0m4 (ogg) -- CONFIRMED (not [ASSUMED]) against the +# authoritative public source https://lco.global/observatory/sites/mpccodes/ (SITEID +# column combined with the first 3 chars of TELID, deduped across all rows); this is +# stronger evidence than the operator-confirmation basis for the original Plan 07-01 +# entries above. Closes the SITE_TELESCOPE_MAP completeness gap found in Phase 7 UAT +# Test 1 (07-UAT.md Gaps section), where a real placed record (observation_id=4213127) +# resolved to ('coj', '1m0') but fell back to [UNVERIFIED] for lack of this entry. +SITE_TELESCOPE_MAP = { + ('coj', '2m0'): 'COJ-2m0', + ('coj', '1m0'): 'COJ-1m0', + ('coj', '0m4'): 'COJ-0m4', + ('ogg', '2m0'): 'OGG-2m0', + ('ogg', '0m4'): 'OGG-0m4', + ('sor', '4m0'): 'SOR-4m0', + ('elp', '1m0'): 'ELP-1m0', + ('elp', '0m4'): 'ELP-0m4', + ('lsc', '1m0'): 'LSC-1m0', + ('lsc', '0m4'): 'LSC-0m4', + ('cpt', '1m0'): 'CPT-1m0', + ('cpt', '0m4'): 'CPT-0m4', + ('tfn', '1m0'): 'TFN-1m0', + ('tfn', '0m4'): 'TFN-0m4', +} + +# SYNC-08/D-10: explicit timeout, single attempt, no retry/backoff loop. This is the +# first explicit HTTP timeout introduced anywhere in solsys_code/ -- there is no +# existing precedent to follow (JPLSBDBQuery.run_query() calls requests.get() with no +# timeout at all, a known anti-pattern, not a convention to mirror here). +_API_TIMEOUT_SECONDS = 10 + +# EXTRACT-01/D-01: configuration_type values that mark a config as the scientifically +# meaningful one, as opposed to a calibration config (ARC/LAMP_FLAT, SOAR) or an +# NRES-specific config (never in scope). Confirmed against installed tom_observations: +# ocs.py:1025-1030/1213 (flat c_{N}_configuration_type key), lco.py:740-743,757-760,998 +# (EXPOSE/REPEAT_EXPOSE/SPECTRUM/REPEAT_SPECTRUM), soar.py:103,118 (SPECTRUM/ARC/ +# LAMP_FLAT), blanco.py:177 (STANDARD -- vocabulary adopted now for forward +# compatibility per CONTEXT.md D-01; Blanco facility scope itself stays deferred). +_SCIENCE_CONFIGURATION_TYPES = {'EXPOSE', 'REPEAT_EXPOSE', 'SPECTRUM', 'REPEAT_SPECTRUM', 'STANDARD'} + +# D-04: LCO MUSCAT records have no flat c_N_exposure_time, only per-channel +# c_N_ic_M_exposure_time_{suffix} keys (confirmed lco.py:585-596, +# LCOMuscatImagingObservationForm). Detect population by ANY of the 4 channels being +# truthy -- more lenient than the real submission form's all-4-required validation. +_MUSCAT_CHANNEL_SUFFIXES = ('g', 'r', 'i', 'z') + + +class InstrumentExtractionError(Exception): + """Raised when _extract_instrument finds no usable config (D-06 total extraction failure). + + Caught separately in handle() so a fully-malformed record is routed to the + dedicated 'extraction_failed' counter, never silently merged into 'skipped'. + """ + + +def _aperture_class_from_telescope_code(telescope_code: str | None) -> str | None: + """Extract the aperture-class token (D-04 vocabulary) from a 4-char telescope code. + + Args: + telescope_code: e.g. '0m4b', '1m0a', '2m0a' (from the API response's + 'telescope' key) -- a 3-char aperture-class token plus a trailing + dome-instance letter suffix. May be None if a malformed/tampered + API block omitted the 'telescope' key (T-07-03); routes to fallback. + + Returns: + str | None: '0m4'/'1m0'/'2m0'/'4m0' (strips the trailing dome-instance + letter), or None if telescope_code is None or doesn't match the + expected 3-char-class + 1-char-suffix shape (routes the caller to + fallback per TELESCOPE-03). Never raises. + """ + if not telescope_code: + return None + if len(telescope_code) >= 4 and telescope_code[:3] in {'0m4', '1m0', '2m0', '4m0'}: + return telescope_code[:3] + return None + + +def _derive_telescope(site: str | None, telescope_code: str | None) -> str | None: + """Map a resolved (site, telescope_code) pair to a verified label via SITE_TELESCOPE_MAP. + + Args: + site: 3-letter site code from the API response (e.g. 'lsc'). May be + None if a malformed/tampered API block omitted the 'site' key + (T-07-03); routes to fallback. + telescope_code: 4-char telescope code from the API response (e.g. + '1m0a'). May be None for the same reason; routes to fallback. + + Returns: + str | None: the verified label (e.g. 'LSC-1m0'), or None if either + site or telescope_code is None, the (site, class) pair isn't in + SITE_TELESCOPE_MAP, or telescope_code's aperture class couldn't be + parsed -- caller falls back to the coarse instrument-class label + (TELESCOPE-03). Never raises. + """ + aperture_class = _aperture_class_from_telescope_code(telescope_code) + if aperture_class is None: + return None + return SITE_TELESCOPE_MAP.get((site, aperture_class)) + + +def _resolve_placement_block(observation_id: str, facility: LCOFacility) -> dict[str, Any] | None: + """Call the LCO Observation Portal API once to resolve a placed record's block. + + Issues a single, timeout-bounded GET to /api/requests/{observation_id}/observations/ + and selects the same COMPLETED-first-else-PENDING block that + OCSFacility.get_observation_status() selects for scheduled_start/scheduled_end, so + telescope resolution and timing always come from the same block (Pitfall 3). + + Args: + observation_id: the record's LCO observation_id. + facility: a shared LCOFacility/SOARFacility instance (for portal_url/api_key + settings and auth header construction). + + Returns: + dict[str, Any] | None: the matched block dict (with 'site'/'enclosure'/ + 'telescope'/'state' keys) on success, or None if the API call failed, + timed out, or returned no usable COMPLETED/PENDING block. Never raises -- + every failure mode (network error, library auth/validation exception, + malformed/non-JSON body, missing 'state' key) is caught and converted to + None so the caller always falls through to the coarse fallback (SYNC-07: + a per-record failure never aborts the run). The except clause never + references, stringifies, or logs the caught exception (SYNC-09/D-11) -- + ImproperCredentialsException/forms.ValidationError embed response.content + directly and must never be logged verbatim. + """ + try: + response = make_request( + 'GET', + urljoin( + facility.facility_settings.get_setting('portal_url'), + f'/api/requests/{observation_id}/observations/', + ), + headers=facility._portal_headers(), + timeout=_API_TIMEOUT_SECONDS, + ) + blocks = response.json() + except (requests.exceptions.RequestException, ImproperCredentialsException, forms.ValidationError, ValueError): + return None + + if not isinstance(blocks, list): + return None + + current_block = None + for block in blocks: + if block.get('state') == 'COMPLETED': + current_block = block + break + elif block.get('state') == 'PENDING': + current_block = block + return current_block + + +def _has_muscat_exposure_signal(parameters: dict[str, Any], n: int) -> bool: + """Check whether config c_{n} has a populated MUSCAT per-channel exposure key (D-04). + + Args: + parameters: the record's parameters dict. + n: config index (1-5). + + Returns: + bool: True if any of c_{n}_ic_1_exposure_time_{g,r,i,z} is truthy. + """ + return any(parameters.get(f'c_{n}_ic_1_exposure_time_{suffix}') for suffix in _MUSCAT_CHANNEL_SUFFIXES) + + +def _find_science_config(parameters: dict[str, Any]) -> int | None: + """Scan c_1..c_5 for the first config whose configuration_type is a science type (D-01). + + Args: + parameters: the record's parameters dict. + + Returns: + int | None: the config index (1-5) of the first config whose + c_{n}_configuration_type is in _SCIENCE_CONFIGURATION_TYPES, or None if no + config has a recognized science configuration_type. + """ + for n in range(1, 6): + configuration_type = parameters.get(f'c_{n}_configuration_type') + if configuration_type in _SCIENCE_CONFIGURATION_TYPES: + return n + return None + + +def _find_exposure_signal_config(parameters: dict[str, Any]) -> int | None: + """Scan c_1..c_5 for the first config with a populated exposure signal (D-02 fallback). + + Args: + parameters: the record's parameters dict. + + Returns: + int | None: the config index (1-5) of the first config with a truthy flat + c_{n}_exposure_time, or (D-04) a populated MUSCAT per-channel exposure key, + or None if no config has any exposure signal at all. + """ + for n in range(1, 6): + if parameters.get(f'c_{n}_exposure_time') or _has_muscat_exposure_signal(parameters, n): + return n + return None + + +def _extract_instrument(parameters: dict[str, Any]) -> str | None: + """Extract the scientifically meaningful instrument_type from a record's parameters. + + Scans the real c_1..c_5-prefixed multi-configuration shape (D-01..D-06): first by + configuration_type whitelist (science vs. SOAR calibration/NRES configs), falling + back to the first config with a populated exposure signal (flat or MUSCAT + per-channel) if no config has a recognized configuration_type. If no c_N_* config + exists at all (today's legacy single-config shape, pre-dating the c_N_* fields), + falls back to the flat 'instrument_type' key itself -- D-02's "original EXTRACT-01 + heuristic" applied to the degenerate single-config case. + + Args: + parameters: the record's parameters dict. + + Returns: + str | None: the selected config's c_{n}_instrument_type value (D-03, unchanged + in format), the flat 'instrument_type' value for the legacy shape, or None + if neither signal selects any config and no flat key is present (D-06 total + extraction failure -- the caller routes this to a dedicated counter, never + the existing 'skipped' counter). + """ + n = _find_science_config(parameters) + if n is None: + n = _find_exposure_signal_config(parameters) + if n is not None: + return parameters.get(f'c_{n}_instrument_type') + return parameters.get('instrument_type') + + +def _coarse_telescope_label(instrument_type: str, facility_name: str) -> str: + """Derive the coarse aperture-class fallback label from instrument_type and facility. + + LCO instrument type codes are prefixed with the aperture class token (e.g. + '1M0-SCICAM-SINISTRO', '0M4-SCICAM-SBIG', '2M0-SPECTRAL-AG' -- confirmed + lco.py:792), mirroring the installed library's own + `self._get_instruments()[instrument_type]['class']` convention of treating + instrument type as implying aperture class. SOAR instrument type codes (e.g. + 'SOAR_GHTS_REDCAM') do NOT follow this prefix convention, so they never match + and previously fell through to the raw, non-coarse string -- closing the + v1.3 milestone-audit gap (TELESCOPE-03/TELESCOPE-04/SYNC-06): SOAR has exactly + one site and one aperture class per the single `('sor', '4m0')` entry in + SITE_TELESCOPE_MAP, so any SOAR record's fallback label is unconditionally + '4m0', regardless of its raw instrument_type string. + + Args: + instrument_type: the record's extracted instrument_type (D-04 fallback + vocabulary source, e.g. '1M0-SCICAM-SINISTRO', 'SOAR_GHTS_REDCAM'). + facility_name: the record's facility string (`record.facility`, e.g. + 'LCO'/'SOAR') -- NOT an LCOFacility/SOARFacility instance. + + Returns: + str: '4m0' unconditionally if facility_name is SOAR (case-insensitive); + otherwise '0m4'/'1m0'/'2m0' (case-normalized, D-04 vocabulary) if + instrument_type has a recognized leading aperture-class prefix, or the + raw instrument_type string itself if it doesn't -- so the fallback + label is never empty and this never raises. This only affects the + coarse label's text -- it never decides whether a record syncs + (TELESCOPE-03). + """ + if facility_name.upper() == 'SOAR': + return '4m0' + if len(instrument_type) >= 3: + candidate = instrument_type[:3].lower() + if candidate in {'0m4', '1m0', '2m0', '4m0'}: + return candidate + return instrument_type + + +def _update_or_unchanged(event: CalendarEvent, fields: dict[str, Any]) -> tuple[CalendarEvent, str]: + """Apply `fields` to an existing event, saving only if something actually changed. + + Args: + event: the matched CalendarEvent to reconcile. + fields: field-value mapping to set on the event. + + Returns: + tuple[CalendarEvent, str]: (event, 'updated') if any field differed and the + row was saved, or (event, 'unchanged') if every field already matched and + no save was issued (no-churn contract). + """ + changed = [f for f, v in fields.items() if getattr(event, f) != v] + if changed: + for f, v in fields.items(): + setattr(event, f, v) + event.save(update_fields=list(fields.keys()) + ['modified']) + return event, 'updated' + return event, 'unchanged' + + +def insert_or_create_calendar_event( + lookup: dict[str, Any], + fields: dict[str, Any], + *, + start_time_tolerance: timedelta | None = None, +) -> tuple[CalendarEvent, str]: + """Create or update a CalendarEvent, or leave it unchanged if no fields differ. + + Implements the no-churn create-or-update contract shared by all three management + commands (sync_lco_observation_calendar, sync_gemini_observation_calendar, + load_telescope_runs): create a new CalendarEvent if none exists for the given + lookup key, update it in place if any fields changed, or leave it untouched if + nothing changed (SYNC-04 idempotency). + + Args: + lookup: keyword-argument mapping used as the unique lookup key for + CalendarEvent.objects.get_or_create (e.g. {'url': url} for LCO/SOAR + and Gemini sync commands, or {'telescope': ..., 'instrument': ..., + 'start_time': ...} for the load_telescope_runs command). + fields: field-value mapping of CalendarEvent attributes to set when + creating or updating. Not merged with `lookup`; the caller is + responsible for ensuring the combined key+fields set is complete. + start_time_tolerance: if given, the lookup's `start_time` is matched by + proximity (an existing event whose start_time is within +/- this + tolerance of the lookup value counts as the same event) rather than by + exact datetime equality. This exists for load_telescope_runs, whose + `start_time` is a computed sun-event time (telescope_runs.sun_event()) + that drifts by a second or two between independent ingests of the same + (site, night) as astropy's IERS Earth-orientation data is refreshed -- + an exact key would silently create a near-duplicate row on re-ingest. + A proximity WINDOW is used deliberately rather than rounding/truncating + start_time to the minute: any fixed bucketing still splits an event that + drifts across a bucket boundary, whereas a centred window never does. The + URL-keyed sync callers pass None and keep exact-equality behaviour + unchanged. Ignored when the lookup has no `start_time` key. + + Returns: + tuple[CalendarEvent, str]: (event, action) where action is one of + 'created' (new record written), 'updated' (existing record changed + and saved), or 'unchanged' (existing record matched all fields; no + save issued). Callers own counter updates and any sidecar writes. + """ + if start_time_tolerance is not None and 'start_time' in lookup: + start_time = lookup['start_time'] + key = {k: v for k, v in lookup.items() if k != 'start_time'} + window = (start_time - start_time_tolerance, start_time + start_time_tolerance) + # order_by makes the match deterministic; in this domain at most one event + # per (telescope, instrument) ever falls in the window (nights are ~24h apart, + # far wider than any plausible IERS-driven drift). + existing = CalendarEvent.objects.filter(**key, start_time__range=window).order_by('start_time').first() + if existing is not None: + # start_time itself is intentionally NOT in `fields`: leaving the stored + # (first-ingested) value pinned avoids churning `modified` every re-ingest + # just because the recomputed sun-event time drifted within tolerance. + return _update_or_unchanged(existing, fields) + return CalendarEvent.objects.create(**lookup, **fields), 'created' + + event, created = CalendarEvent.objects.get_or_create(**lookup, defaults=fields) + if created: + return event, 'created' + return _update_or_unchanged(event, fields) diff --git a/solsys_code/campaign_filters.py b/solsys_code/campaign_filters.py new file mode 100644 index 00000000..7466e97c --- /dev/null +++ b/solsys_code/campaign_filters.py @@ -0,0 +1,28 @@ +"""django-filter FilterSet for the per-campaign CampaignRun read path (VIEW-04). + +``run_status`` must be explicitly declared as a ``MultipleChoiceFilter`` -- ``Meta.fields`` +auto-generation produces a single-value ``CharFilter`` for a ``choices`` ``CharField`` (no +special-casing for ``choices`` in django-filter's ``FILTER_FOR_DBFIELD_DEFAULTS``), which would +violate D-12's OR-semantics multi-select requirement. ``open_to_collaboration`` is a plain +``BooleanField`` and is left to ``Meta.fields`` auto-generation, which correctly produces a +``BooleanFilter``. +""" + +import django_filters +from django import forms + +from .models import CampaignRun + + +class CampaignRunFilterSet(django_filters.FilterSet): + """VIEW-04: multi-select run_status (OR semantics, D-12) + boolean open_to_collaboration.""" + + run_status = django_filters.MultipleChoiceFilter( + choices=CampaignRun.RunStatus.choices, + label='Run status', + widget=forms.CheckboxSelectMultiple, + ) + + class Meta: # noqa: D106 + model = CampaignRun + fields = ['run_status', 'open_to_collaboration'] diff --git a/solsys_code/campaign_forms.py b/solsys_code/campaign_forms.py new file mode 100644 index 00000000..9737056f --- /dev/null +++ b/solsys_code/campaign_forms.py @@ -0,0 +1,177 @@ +"""Public-facing campaign run submission form (SUBMIT-01/SUBMIT-04, D-05/D-06). + +A plain `forms.Form` -- NEVER a `ModelForm`. `CampaignRun.telescope_instrument` has no +`blank=True` on the model, so a `ModelForm` would derive `required=True` from the model field and +wrongly force it required, contradicting D-05 ("everything except `campaign` is optional"). +Explicit `required=False` on every non-`campaign` field sidesteps this entirely. +""" + +from crispy_forms.bootstrap import FormActions +from crispy_forms.helper import FormHelper +from crispy_forms.layout import HTML, Div, Fieldset, Layout, Submit +from django import forms +from django.urls import reverse_lazy +from tom_targets.models import Target, TargetList + +from solsys_code.campaign_utils import parse_obs_window +from solsys_code.solsys_code_observatory.models import Observatory + + +class CampaignRunSubmissionForm(forms.Form): + """Public intake form for a single campaign observing run, pending staff review.""" + + campaign = forms.ModelChoiceField(queryset=TargetList.objects.all(), required=True) + telescope_instrument = forms.CharField(max_length=255, required=False, label='Telescope / instrument') + # D-09: live-search widget, no create-new-site link (public submitters never get a + # site-creation path; unmatched free text is allowed and never blocks submission). + # NOTE (htmx hx-trigger grammar): the `[...]` event filter goes IMMEDIATELY AFTER the + # event name, with modifiers (`changed`, `delay:300ms`) following -- 22-REVIEWS.md + # finding 1. Do NOT reorder this to `input changed delay:300ms[...]`; htmx does not + # parse a filter placed after the modifiers. + site_raw = forms.CharField( + max_length=255, + required=False, + label='Observing site', + widget=forms.TextInput( + attrs={ + 'hx-get': reverse_lazy('campaigns:site_search'), + 'hx-trigger': 'input[this.value.length >= 2] changed delay:300ms', + 'hx-target': '#site-suggestions-id_site_raw', + 'hx-swap': 'innerHTML', + 'hx-vals': '{"input_id": "id_site_raw"}', + 'autocomplete': 'off', + 'placeholder': 'MPC code or site name…', + 'class': 'form-control', + } + ), + ) + # A3: collapses to a single observing-date free-text field -- the window schema has no + # time-of-night component, so the UT start/end DateTimeField inputs have no home here + # and are dropped entirely (not repurposed). This is now free text parsed by + # `parse_obs_window()` (SUBMIT-01 date-format gap fix) rather than a strict single + # `DateField` -- clean() below maps the parsed window onto cleaned_data['window_start']/ + # ['window_end'], which the view reads (single-night collapse or a genuine multi-night + # range; blank -> TBD, both None). + obs_date = forms.CharField( + required=False, + max_length=255, + label='Observation date', + help_text=( + 'A single date (YYYY-MM-DD), a date range (YYYY-MM-DD -- YYYY-MM-DD or ' + 'YYYY-MM-DD to YYYY-MM-DD), or leave blank if not yet scheduled (TBD).' + ), + ) + filters_bandpass = forms.CharField(max_length=255, required=False, label='Filter(s) / bandpass') + observation_details = forms.CharField(widget=forms.Textarea, required=False, label='Observation details') + open_to_collaboration = forms.BooleanField(required=False, label='Open to collaboration?') + contact_person = forms.CharField(max_length=255, required=True, label='Contact person') # D-06 + contact_email = forms.EmailField(required=True, label='Contact email') # D-06 + # VIEW-05/D-07: default-opt-out combined contact-visibility flag. required=False means an + # unchecked box (the default) cleans to False. + contact_public_opt_in = forms.BooleanField( + required=False, + label='Show contact info publicly?', + help_text=( + 'If checked, your name and email will be shown on the public campaign table. ' + 'Leave unchecked to keep them visible to staff only (default).' + ), + ) + comments = forms.CharField(widget=forms.Textarea, required=False, label='Other comments') + # SUBMIT-04: hidden honeypot, non-obvious name, never rendered visibly to a human. + alt_contact_info = forms.CharField(required=False, widget=forms.HiddenInput()) + + def clean_alt_contact_info(self): + """Never raise -- SUBMIT-04: a tripped bot must get no error signal. The view (Plan 02) + decides what to do with a filled value; the form only passes it through. + """ + return self.cleaned_data.get('alt_contact_info', '') + + def clean(self): + """Parse the free-text `obs_date` into a window via `parse_obs_window()`. + + Mirrors `import_campaign_csv`'s needs-review discipline (act on the parser's flag, + never raise) but adapted to Django form-error convention: non-blank unparseable text + surfaces a friendly `obs_date` error (`form.add_error`, not a silent skip); blank text + also comes back with `window_needs_review=True` but is an intentional TBD and must NOT + error. `parse_obs_window()` always returns `window_start`/`window_end` both `None` or + both set (single-night collapse or a genuine range), never one-`None` -- this keeps the + model's `campaign_run_window_start_end_null_together` CheckConstraint invariant intact + with no extra code needed here. There is no UT-time field on this public form (per the + `obs_date` field's A3 comment), so an empty `ut_range_raw` is passed through. + """ + cleaned_data = super().clean() + obs_date_raw = cleaned_data.get('obs_date', '') or '' + ( + window_start, + window_end, + _original_raw, + window_needs_review, + _ut_start, + _ut_end, + _ut_needs_review, + ) = parse_obs_window(obs_date_raw, '') + cleaned_data['window_start'] = window_start + cleaned_data['window_end'] = window_end + if window_needs_review and obs_date_raw.strip(): + self.add_error( + 'obs_date', + "Couldn't understand this date. Use a single date (YYYY-MM-DD), a range " + '(YYYY-MM-DD -- YYYY-MM-DD or YYYY-MM-DD to YYYY-MM-DD), or leave it blank ' + 'if the observing date is not yet scheduled.', + ) + return cleaned_data + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.helper = FormHelper() + self.helper.layout = Layout( + 'campaign', + Fieldset( + 'Run details', + 'telescope_instrument', + 'site_raw', + HTML('
'), + 'obs_date', + 'filters_bandpass', + 'observation_details', + 'open_to_collaboration', + ), + Fieldset('Contact', 'contact_person', 'contact_email', 'contact_public_opt_in', 'comments'), + # Hidden via widget=HiddenInput above; belt-and-suspenders CSS hiding too. + Div('alt_contact_info', css_class='d-none'), + FormActions(Submit('submit', 'Submit run for review')), + ) + + +class CampaignGapAnalysisForm(forms.Form): + """Campaign-scoped target/site/date-range selection form for coverage-gap analysis (GAP-02). + + A plain `forms.Form` -- NOT a `ModelForm` -- matching `CampaignRunSubmissionForm`'s style. + The `target`/`site` querysets MUST be scoped to a specific campaign at instantiation time + (via `campaign=` in `__init__`), never via a class-level unscoped queryset (D-12/D-13's + dropdown-population rules). The view (`CampaignGapAnalysisView`) re-validates any submitted + `target`/`site` pk server-side regardless of what these querysets offer -- this form only + controls what's *offered*, not what a raw request can submit (Pitfall 3, IDOR). + """ + + target = forms.ModelChoiceField(queryset=Target.objects.none(), required=False, label='Target') + site = forms.ModelChoiceField(queryset=Observatory.objects.none(), required=True, label='Site') + end_date = forms.DateField(required=False, label='End date (optional)') + + def __init__(self, *args, campaign=None, **kwargs): + super().__init__(*args, **kwargs) + if campaign is not None: + # D-12: target is optional (auto-selected server-side) for a single-target + # campaign; required when there's more than one target to disambiguate. + self.fields['target'].queryset = campaign.targets.all() + self.fields['target'].required = campaign.targets.count() > 1 + # D-13: only Observatory records actually used by this campaign's CampaignRuns. + self.fields['site'].queryset = Observatory.objects.filter(campaign_runs__campaign=campaign).distinct() + self.helper = FormHelper() + self.helper.form_method = 'get' # D-09: gap analysis is a plain GET, not an htmx POST + self.helper.layout = Layout( + 'target', + 'site', + 'end_date', + FormActions(Submit('submit', 'Update Results')), + ) diff --git a/solsys_code/campaign_gap.py b/solsys_code/campaign_gap.py new file mode 100644 index 00000000..17fe3246 --- /dev/null +++ b/solsys_code/campaign_gap.py @@ -0,0 +1,278 @@ +"""Pure-logic core of the coverage-gap analysis feature (GAP-01/GAP-02). + +Composes ``telescope_runs.sun_event()`` (the observable side, dark-window-only per +``17-GAP-01-DECISION.md``) with a ``CampaignRun`` query (the claimed side) into a set +difference, cached via Django's low-level cache framework with a 1-hour TTL. Mirrors +``campaign_utils.py``'s role: a pure-logic helper module with no view/request concerns, +structured with the same "never raise for expected messy data" discipline. + +This module depends only on the heavy SPICE-loading ephemeris module's read-only, +already-tested sun-event helper for its ephemeris needs -- it must never import the heavy +SPICE-loading ephemeris module (or any module that imports it, such as ``solsys_code.views``) +at module scope. That module's ~1.6 GB SPICE-kernel download side effect (CLAUDE.md "Heavy +import side effect") would otherwise be paid by every process that imports this module. +""" + +import logging +from datetime import date, timedelta + +from django.core.cache import cache +from django.utils import timezone + +from solsys_code.models import CampaignRun +from solsys_code.solsys_code_observatory.models import Observatory +from solsys_code.telescope_runs import sun_event + +logger = logging.getLogger(__name__) + +GAP_CACHE_TTL_SECONDS = 3600 # D-10: 1-hour result cache +DEFAULT_WINDOW_DAYS = 90 # D-11: default date-range window +MAX_WINDOW_DAYS = 180 # D-11: hard cap on requested date-range span + +# D-05: a CampaignRun in one of these run_status values never "claims" a date, even if +# approval_status=APPROVED -- a run that fell through in the real world frees its date +# back up as a gap. +_EXCLUDED_RUN_STATUSES = frozenset( + { + CampaignRun.RunStatus.CANCELLED, + CampaignRun.RunStatus.NOT_AWARDED, + CampaignRun.RunStatus.WEATHER_TECH_FAILURE, + } +) + + +def clamp_date_range(today: date, requested_end: date | None) -> tuple[date, date]: + """Enforce D-11's 90-day default / 180-day max span, independent of client input. + + Args: + today: the local "today" the range starts from (always the start of the range). + requested_end: a client-supplied end date, or None to use the 90-day default. + + Returns: + tuple[date, date]: (start, end), where start is always `today` and end is never + later than `today + MAX_WINDOW_DAYS` days, regardless of `requested_end`. + """ + start = today + default_end = start + timedelta(days=DEFAULT_WINDOW_DAYS) + max_end = start + timedelta(days=MAX_WINDOW_DAYS) + if requested_end is None: + return start, default_end + # WR-02: also floor at `start` -- otherwise a past `requested_end` (e.g. a client + # submitting end_date=2020-01-01) produces end < start, an empty range, and a + # misleading "no gaps found" instead of reflecting that nothing was actually searched. + return start, max(start, min(requested_end, max_end)) + + +def build_gap_cache_key(campaign_pk: int, target_pk: int | None, site_pk: int, start: date, end: date) -> str: + """Build a stable, collision-free cache key for a gap-analysis request (D-10). + + Args: + campaign_pk: pk of the campaign (TargetList). + target_pk: pk of the selected Target, or None for a single-target campaign that + has no per-target disambiguation need (D-12). Encoded explicitly as the + literal 'none' rather than omitted, so a null-target request never collides + with a differently-scoped one (D-10 / Information Disclosure control). + site_pk: pk of the selected Observatory. + start: inclusive start date of the requested range. + end: inclusive end date of the requested range. + + Returns: + str: a delimited cache key including all four dimensions (campaign, target, + site, date range). + """ + target_segment = str(target_pk) if target_pk is not None else 'none' + return f'campaign_gap:{campaign_pk}:{target_segment}:{site_pk}:{start.isoformat()}:{end.isoformat()}' + + +def observable_dates(site, start: date, end: date) -> set[date]: + """Return the set of dates in [start, end] with a non-zero -15 degree dark window. + + D-04: any non-zero dark window counts as observable -- no minimum-duration threshold. + D-03: a per-date `sun_event(kind='dark')` ValueError (e.g. a hypothetical future + polar/midnight-sun Observatory) skips that one date as "unknown"; it never aborts the + rest of the loop, matching this codebase's established per-record log+skip discipline. + + Args: + site: an Observatory instance (sun_event() accepts any Observatory, not just a + SITES-dict-registered one). + start: inclusive start date. + end: inclusive end date. + + Returns: + set[date]: dates whose dark window is non-zero. + """ + observable = set() + n_days = (end - start).days + 1 + for i in range(n_days): + d = start + timedelta(days=i) + try: + sun_event(site, d, kind='dark') + observable.add(d) + except ValueError: + logger.debug('sun_event(dark) raised for site=%s date=%s; skipping as unknown (D-03).', site, d) + return observable + + +def claimed_dates(campaign, target, site) -> tuple[set[date], list, list, list]: + """Return the set of dates claimed by approved, non-terminal-failure CampaignRuns. + + D-05: a date is claimed when a CampaignRun has approval_status=APPROVED and + run_status is not in {cancelled, not_awarded, weather_tech_failure}. + + Target attribution (Pitfall 4 / D-12): if the campaign has exactly one Target, the + query does NOT filter by target -- the single target is implied, and real imported + runs commonly have target=None (per import_campaign_csv's single-target + auto-assignment precedent). If the campaign has more than one Target, the query + filters target= strictly, and target=None rows are collected into a + separate "unattributed" list rather than being counted as claiming (or not claiming) + any specific target's dates -- a data-quality signal, not a silent guess either way. + + Ground-vs-space asset-awareness (ASSET-01/ASSET-02, D-09): the classification is + computed once, before the loop, from the ``site`` parameter (``site.observations_type + == Observatory.SATELLITE_OBSTYPE``) -- never re-read per-row (Pitfall 3), since the + queryset is already filtered to this single site. For a ground run, every date in the + inclusive range [window_start, window_end] is claimed (a single-night run has + window_start == window_end, so exactly one date is claimed). A space-mission run whose + window hasn't narrowed to a single night (window_start != window_end) claims nothing + and is collected into a separate "pending narrowing" list instead -- a space + observatory has no fixed horizon, so claiming every date in a broad window would + wrongly mark those nights as covered. A run with window_start is None (TBD) cannot be + attributed to any date regardless of site type -- it is collected into a separate + "undated" list, never added to the claimed set and never added to "pending narrowing" + (D-09 explicit distinction: "no info at all" vs. "a real space-mission run with a + range, just not scheduled tight enough yet"). + + Args: + campaign: the campaign TargetList. + target: the selected Target, or None. + site: the selected Observatory. + + WR-05: unlike ``observable_dates(site, start, end)``, this function takes no date-range + parameters -- it returns every approved, non-excluded ``CampaignRun`` for the campaign/ + site combination regardless of any requested window. ``_compute_gap()`` only ever + evaluates the range-bounded ``gap = obs - claimed`` against the range-bounded ``obs`` + set, so ``gap_dates`` is correct -- but the returned ``claimed_dates``/``undated_runs``/ + ``unattributed_runs``/``pending_narrowing_runs`` are campaign/site-wide, NOT scoped to + ``[start, end]``, even though the cached result they end up in + (``build_gap_cache_key()``) is keyed by a date range. Do not assume a range-keyed cache + entry's ``claimed_dates`` is itself range-bounded. + + Returns: + tuple[set[date], list, list, list]: (claimed_dates, undated_runs, + unattributed_runs, pending_narrowing_runs). + """ + # D-13/WR-01: restrict the columns actually fetched to a PII-free field set (never + # contact_person/contact_email) before anything is collected into + # `undated_runs`/`unattributed_runs` and cached -- mirrors CampaignRunTableView's + # "restrict the queryset, not just the rendered output" discipline. `.only()` (not + # `.values()`) keeps these as CampaignRun instances so existing pk-based equality and + # attribute access downstream keep working; only pk/window_start/window_end are fetched. + qs = CampaignRun.objects.filter(campaign=campaign, site=site, approval_status=CampaignRun.ApprovalStatus.APPROVED) + qs = qs.exclude(run_status__in=_EXCLUDED_RUN_STATUSES) + qs = qs.only('pk', 'window_start', 'window_end') + + unattributed_runs: list[CampaignRun] = [] + single_target = campaign.targets.count() == 1 + if not single_target: + # Multi-target campaign: target=None rows are ambiguous -- don't count them as + # claiming this specific target's dates, but don't silently drop them either. + unattributed_runs = list(qs.filter(target__isnull=True)) + qs = qs.filter(target=target) + # Single-target campaign: don't filter by target at all -- the single target is + # implied and target=None is the common real-data case (Pitfall 4). + + # ASSET-01: classification computed once from the site parameter, before the loop -- + # never a per-row run.site read (Pitfall 3), which would force widening the + # PII-minimizing .only('pk', 'window_start', 'window_end') queryset above. + is_space_mission = site.observations_type == Observatory.SATELLITE_OBSTYPE + + claimed: set[date] = set() + undated_runs: list[CampaignRun] = [] + pending_narrowing_runs: list[CampaignRun] = [] + for run in qs: + if run.window_start is None or run.window_end is None: + # TBD -- can't be attributed to any date (unchanged bucketing rule), regardless + # of site type (D-09 explicit distinction from pending_narrowing_runs below). + # WR-02: also catches the DB-CheckConstraint-should-prevent-but-defend-anyway + # case of a mismatched pair (one set, one NULL) so this never raises a + # TypeError on read. + undated_runs.append(run) + continue + if is_space_mission and run.window_start != run.window_end: + # ASSET-02/D-09: a space-mission run with an un-narrowed range claims nothing + # until a staff edit or CSV re-import narrows it to window_start == window_end + # (D-10: no automated narrowing mechanism). + pending_narrowing_runs.append(run) + continue + n_days = (run.window_end - run.window_start).days + 1 + for i in range(n_days): + claimed.add(run.window_start + timedelta(days=i)) + + return claimed, undated_runs, unattributed_runs, pending_narrowing_runs + + +def _compute_gap(campaign, target, site, start: date, end: date) -> dict: + """Compute the coverage-gap result dict (no caching). + + Args: + campaign: the campaign TargetList. + target: the selected Target, or None. + site: the selected Observatory. + start: inclusive start date. + end: inclusive end date. + + Returns: + dict: gap_dates, claimed_dates, observable_dates (each a sorted list of dates), + undated_runs, unattributed_runs, pending_narrowing_runs (lists of CampaignRun), + and unknown_date_count (number of dates in range whose sun_event() call + raised, i.e. dates in range that are neither observable nor + known-unavailable). + """ + obs = observable_dates(site, start, end) + claimed, undated_runs, unattributed_runs, pending_narrowing_runs = claimed_dates(campaign, target, site) + gap = obs - claimed + + n_days = (end - start).days + 1 + # observable_dates() only ever adds a date when sun_event() succeeds (D-04: any + # non-zero dark window -- i.e. any successful 2-crossing evaluation -- counts as + # observable), so every date in range that did NOT end up in `obs` is exactly a date + # whose sun_event() call raised ValueError (D-03) and was skipped as unknown. + unknown_date_count = n_days - len(obs) + + return { + 'gap_dates': sorted(gap), + 'claimed_dates': sorted(claimed), + 'observable_dates': sorted(obs), + 'undated_runs': undated_runs, + 'unattributed_runs': unattributed_runs, + 'pending_narrowing_runs': pending_narrowing_runs, + 'unknown_date_count': unknown_date_count, + } + + +def get_or_compute_gap(campaign, target, site, start: date, end: date) -> dict: + """Cache-or-compute wrapper for the coverage-gap result (D-10). + + On a cache hit, returns the cached dict unchanged -- its original `computed_at` must + survive so the "last computed at" display reflects when the result was actually + computed, not the time of this (cache-hit) request. On a cache miss, computes the + result, stamps `computed_at`, caches it for GAP_CACHE_TTL_SECONDS, and returns it. + + Args: + campaign: the campaign TargetList. + target: the selected Target, or None. + site: the selected Observatory. + start: inclusive start date. + end: inclusive end date. + + Returns: + dict: see `_compute_gap`'s return value, plus a `computed_at` key. + """ + key = build_gap_cache_key(campaign.pk, target.pk if target else None, site.pk, start, end) + cached = cache.get(key) + if cached is not None: + return cached + result = _compute_gap(campaign, target, site, start, end) + result['computed_at'] = timezone.now() + cache.set(key, result, timeout=GAP_CACHE_TTL_SECONDS) + return result diff --git a/solsys_code/campaign_tables.py b/solsys_code/campaign_tables.py new file mode 100644 index 00000000..32a8b6f8 --- /dev/null +++ b/solsys_code/campaign_tables.py @@ -0,0 +1,377 @@ +"""django-tables2 Table definition for the per-campaign CampaignRun read path (VIEW-01). + +Renders identically whether ``record`` is a plain ``dict`` (the restricted ``.values()`` +queryset used for non-staff requests, D-13/VIEW-03) or a full ``CampaignRun`` model instance +(staff requests). django-tables2's automatic ``get_FOO_display()`` choice-label lookup is +skipped for dict rows (15-RESEARCH.md Pitfall 2), so ``run_status``/``approval_status`` labels +are resolved manually here via the model's ``TextChoices`` rather than relied on automatically. +""" + +import django_tables2 as tables +from django.middleware.csrf import get_token +from django.urls import reverse +from django.utils.html import format_html +from django.utils.http import urlencode +from django_tables2.utils import Accessor + +from .campaign_utils import is_placeholder_observatory +from .models import CampaignRun + +# D-08 / UI-SPEC Approval-Status Badge Contract: fixed 3-entry dict, badge class never derived +# from the raw DB string (mirrors calendar_display_extras.py's constant-lookup pattern shape). +APPROVAL_BADGE_CLASSES = { + CampaignRun.ApprovalStatus.PENDING_REVIEW: 'badge-warning', + CampaignRun.ApprovalStatus.APPROVED: 'badge-success', + CampaignRun.ApprovalStatus.REJECTED: 'badge-danger', +} + +# UI-SPEC Run-Status Badge Contract: deliberately muted so it never competes with the +# mandatory approval_status badge. Dead-end outcomes use badge-light (+ grey border added in +# render_run_status), NOT badge-danger -- danger-red is reserved exclusively for +# approval_status=rejected (see UI-SPEC rationale). +RUN_STATUS_BADGE_CLASSES = { + CampaignRun.RunStatus.REQUESTED: 'badge-secondary', + CampaignRun.RunStatus.PLANNED: 'badge-secondary', + CampaignRun.RunStatus.OBSERVED: 'badge-info', + CampaignRun.RunStatus.REDUCED: 'badge-info', + CampaignRun.RunStatus.PUBLISHED: 'badge-primary', + CampaignRun.RunStatus.CANCELLED: 'badge-light', + CampaignRun.RunStatus.NOT_AWARDED: 'badge-light', + CampaignRun.RunStatus.WEATHER_TECH_FAILURE: 'badge-light', +} + +# Fields whose staff-vs-anonymous underlying key genuinely differs (dict path selects +# 'site__short_name' explicitly, never 'site' -- see campaign_views.ALLOWED_FIELDS_FOR_NON_STAFF), +# so the column needs an Accessor that resolves both a literal dict key and a model-instance +# attribute chain (confirmed against installed django_tables2.utils.Accessor.resolve source). +_FREE_TEXT_ATTRS = {'td': {'class': 'text-truncate', 'style': 'max-width: 200px;'}} + + +class CampaignRunTable(tables.Table): + """Spreadsheet-parity CampaignRun table (D-09), PII-gated via the view's ``exclude=`` kwarg.""" + + site = tables.Column(accessor='site__short_name', verbose_name='Site', empty_values=()) + + class Meta: # noqa: D106 + model = CampaignRun + fields = ( + 'telescope_instrument', + 'site', + 'window_start', + 'filters_bandpass', + 'run_status', + 'approval_status', + 'open_to_collaboration', + 'observation_details', + 'weather', + 'observation_outcome', + 'publication_plans', + 'comments', + 'contact_person', + 'contact_email', + ) + template_name = 'django_tables2/bootstrap4-responsive.html' + attrs = {'class': 'table table-bordered table-sm'} + empty_text = 'No runs match these filters. Clear filters to see all runs for this campaign.' + + observation_details = tables.Column(attrs=_FREE_TEXT_ATTRS) + weather = tables.Column(attrs=_FREE_TEXT_ATTRS) + observation_outcome = tables.Column(attrs=_FREE_TEXT_ATTRS) + publication_plans = tables.Column(attrs=_FREE_TEXT_ATTRS) + comments = tables.Column(attrs=_FREE_TEXT_ATTRS) + + def render_run_status(self, record): + """Render run_status as a muted Bootstrap badge (UI-SPEC Run-Status Badge Contract). + + Reads the raw stored value from ``record`` via Accessor rather than accepting + django-tables2's pre-resolved ``value`` kwarg: for model-instance rows (staff), + django-tables2's row machinery auto-calls ``get_run_status_display()`` *before* + this method runs (since the field has ``choices``), silently handing us the + already-humanized label instead of the raw code -- would break the + ``CampaignRun.RunStatus(value)`` lookup below. Resolving from ``record`` directly + sidesteps that pre-processing and gives the raw code for both dict and model rows. + """ + value = Accessor('run_status').resolve(record, quiet=True) + css = RUN_STATUS_BADGE_CLASSES.get(value, 'badge-secondary') + label = CampaignRun.RunStatus(value).label + style = 'border: 1px solid #6c757d;' if css == 'badge-light' else '' + return format_html('{}', css, style, label) + + def render_approval_status(self, record): + """Render approval_status as a colored Bootstrap badge (D-08). + + See render_run_status docstring -- same raw-value-via-Accessor rationale applies. + """ + value = Accessor('approval_status').resolve(record, quiet=True) + css = APPROVAL_BADGE_CLASSES.get(value, 'badge-secondary') + label = CampaignRun.ApprovalStatus(value).label + return format_html('{}', css, label) + + def render_site(self, record): + """Show Observatory.short_name when resolved, else the submitted site_raw text. + + Falls back to ``site_raw`` whenever the site is unresolved (``site__short_name`` + empty) and ``site_raw`` is non-empty, regardless of ``site_needs_review`` -- + pending runs (D-07) leave ``site_needs_review`` False until approval, so relying + on that flag alone hid every pending submission's site text from staff. When + resolution genuinely ran and failed (``site_needs_review`` True), keep the + failure styling (warning triangle); otherwise (not yet attempted) show a plain + muted-italic "pending review" presentation with no failure icon. + """ + site_short_name = Accessor('site__short_name').resolve(record, quiet=True) + if site_short_name: + return site_short_name + site_raw = Accessor('site_raw').resolve(record, quiet=True) or '' + if not site_raw: + return '' + if Accessor('site_needs_review').resolve(record, quiet=True): + return format_html( + '' + ' {}', + site_raw, + ) + return format_html( + '{}', + site_raw, + ) + + def render_window_start(self, record): + """Render the observing window as a TBD badge, single date, or 'start -> end' range. + + Resolves both window_start/window_end via Accessor (D-03/D-05) so this works + identically whether record is a dict (non-staff) or a CampaignRun instance + (staff) -- mirrors render_site()'s dict-vs-model dual-accessor precedent. + + The TBD badge additionally carries a ``title`` tooltip with + ``original_obs_date_raw`` (D-08) when that field is non-empty, so staff can see + exactly what the sheet said without new display machinery -- reuses render_site()'s + format_html tooltip convention. The raw text is interpolated as a positional + format_html argument so Django auto-escapes it (mitigates stored-XSS from + community-editable sheet text, T-20-03); never mark_safe or string concatenation. + """ + start = Accessor('window_start').resolve(record, quiet=True) + end = Accessor('window_end').resolve(record, quiet=True) + if start is None: # both null by the model's own invariant + original_obs_date_raw = Accessor('original_obs_date_raw').resolve(record, quiet=True) or '' + if original_obs_date_raw: + return format_html('TBD', original_obs_date_raw) + return format_html('TBD') + if start == end: + return start # single-night row (D-05) + return format_html('{} -> {}', start, end) # D-05: literal "->", not an en-dash + + def render_open_to_collaboration(self, value): + """Render open_to_collaboration as a Yes/No icon (UI-SPEC column set).""" + if value: + return format_html('') + return format_html('') + + +class ApprovalQueueTable(CampaignRunTable): + """CampaignRunTable plus an Actions column for the staff approval queue (D-01/D-02). + + The pending-review table (``show_actions=True``, the default) renders an Approve/Reject + button pair per row that POST directly to ``campaigns:decide``; the recently-decided table + (``show_actions=False``) renders the same columns with an empty Actions cell -- read-only + per 16-RESEARCH.md Open Question 2. CSRF protection is handled inside ``render_actions`` + itself (via ``django.middleware.csrf.get_token``) rather than in the template's row loop, + since ``{% render_table %}`` doesn't hand row-rendering control back to the template (see + 16-03-PLAN.md Task 1 planner note) -- the request object must be passed in explicitly at + construction time so a CSRF token can be minted for each row's mini-forms. + """ + + actions = tables.Column(empty_values=(), orderable=False, verbose_name='Actions') + + # Triage-focused queue view (UAT Test 14 gap closure, 16-05): drop the three + # post-observation columns (weather, observation_outcome, publication_plans) that have + # no CampaignRunSubmissionForm field and are therefore structurally always blank on a + # PENDING_REVIEW row, and front-load `actions` so Approve/Reject is reachable without + # horizontal scrolling. CampaignRunTable itself is untouched -- it stays spreadsheet-parity + # for Phase 15's D-09 read path. + class Meta(CampaignRunTable.Meta): # noqa: D106 + exclude = ('weather', 'observation_outcome', 'publication_plans') + sequence = ( + 'actions', + 'approval_status', + 'telescope_instrument', + 'site', + 'window_start', + '...', + ) + + def __init__( + self, + *args, + show_actions=True, + request=None, + candidate_pool=None, + mode='pending', + status_actions=False, + **kwargs, + ): + self.show_actions = show_actions + self.request = request + self.candidate_pool = candidate_pool + # D-07/D-10: extends the show_actions convention rather than replacing it. + # 'pending' (default) is the existing pending-review row; 'resolve' is the new + # Sites Needing Review row (Task 2). show_actions=False (decided table) still wins + # over either mode in render_site()'s early-return below. + self.mode = mode + # D-04 (Plan 02): independent of show_actions -- gates the Decided table's new + # Mark Cancelled/Mark Weathered action, which must render WITHOUT flipping + # show_actions to True (RESEARCH Pitfall 3: that would also leak the live-search + # site widget into the read-only Decided table's unresolved-site rows). + self.status_actions = status_actions + super().__init__(*args, **kwargs) + + def _render_site_search_widget(self, *, site_raw, input_id, form_id): + """Shared live-search widget markup (hx-get to campaigns:site_search, D-10) used by + both the pending row and the resolve-mode row -- differs only in which form the + input's HTML5 ``form=`` attribute targets. + + NOTE (htmx hx-trigger grammar): the ``[...]`` event filter goes IMMEDIATELY AFTER + the event name, with modifiers (``changed``, ``delay:300ms``) following -- + 22-REVIEWS.md finding 1, same corrected string as the public form's widget. Do NOT + reorder this to ``input changed delay:300ms[...]``; htmx does not parse a filter + placed after the modifiers. Because this trigger string lives in the LITERAL part of + the format_html template (not a substituted argument), it is NOT entity-escaped in + the rendered table. + """ + container_id = f'site-suggestions-{input_id}' + site_search_url = reverse('campaigns:site_search') + create_url = '{}?{}'.format( + reverse('solsys_code_observatory:create'), + urlencode({'obscode': site_raw, 'next': reverse('campaigns:approval_queue')}), + ) + return format_html( + '' + '
' + 'Create new Observatory', + site_raw, + input_id, + form_id, + site_search_url, + container_id, + create_url, + ) + + def render_site(self, record): + """Unresolved actionable row: inline live-search site input (backed by + campaigns:site_search, D-10) + an always-visible "Create new Observatory" link + (SITE-01/D-04), submitted into the row's single decide-form via the HTML5 ``form=`` + attribute. Resolved rows and the read-only decided table (``show_actions=False``) + keep CampaignRunTable's existing plain-text ``render_site`` rendering unchanged. + + A resolve-mode row (``self.mode == 'resolve'``) whose site IS already set to a + REAL (non-placeholder) Observatory is the projection-failed retry state -- + 22-REVIEWS.md finding 8c -- and also keeps the plain-text fallback: such a row's + Resolve button alone re-attempts the projection, no site input is needed. But a + resolve-mode row whose site is a tier-3 PLACEHOLDER Observatory (22-06 gap + closure, UAT gap 2B) falls through to the same live-search widget an unresolved + row gets, since a placeholder is not a genuine resolution and still needs staff + correction. ``not self.show_actions`` suppresses the widget in every case (WR-01, + 22-REVIEW.md): a read-only table must never render live action widgets regardless + of ``self.mode`` or placeholder state. + + Only overridden here (not on ``CampaignRunTable``): only ``ApprovalQueueTable`` + instances carry ``self.show_actions``/``self.candidate_pool``/``self.mode``, so + overriding on the parent would raise ``AttributeError`` for the per-campaign + ``CampaignRunTable``. + """ + site_short_name = Accessor('site__short_name').resolve(record, quiet=True) + if site_short_name: + # 22-06: a set site only falls through to the widget when it's a tier-3 + # placeholder in an actionable resolve-mode row -- everything else (a genuine + # Observatory, any pending-mode row, or a read-only show_actions=False row, + # WR-01) keeps the plain-text render. + site_obj = Accessor('site').resolve(record, quiet=True) + is_correctable_placeholder = ( + self.show_actions and self.mode == 'resolve' and is_placeholder_observatory(site_obj) + ) + if not is_correctable_placeholder: + return super().render_site(record) + elif not self.show_actions: + # WR-01: show_actions must gate resolve-mode rendering too -- a hypothetical + # ApprovalQueueTable(..., mode='resolve', show_actions=False) (e.g. a future + # read-only "resolved sites" audit view) must fall back to the plain-text + # render, not the live search widget, the same way a pending-mode + # show_actions=False table already does. + return super().render_site(record) + pk = Accessor('pk').resolve(record, quiet=True) + site_raw = Accessor('site_raw').resolve(record, quiet=True) or '' + input_id = f'site-input-{pk}' + # Resolve-mode rows submit into their own resolve-form (distinct from the pending + # row's decide-form), matching render_actions()'s resolve-mode form id below. + form_id = f'resolve-form-{pk}' if self.mode == 'resolve' else f'decide-form-{pk}' + return self._render_site_search_widget(site_raw=site_raw, input_id=input_id, form_id=form_id) + + def render_actions(self, record): + """Render one form: Approve/Reject named submit buttons (pending mode), a single + Resolve button (resolve mode, D-08), or nothing (decided-runs table). Single form + (not two) so the Site column's ``form=`` input can target it via the HTML5 ``form=`` + attribute (D-04).""" + if not self.show_actions: + # D-04 (Plan 02): the Decided table's Mark Cancelled/Mark Weathered action is + # gated by the independent status_actions flag, never by flipping show_actions + # -- render_site()'s existing plain-text fallback (the `elif not self.show_actions` + # branch above) stays completely untouched (RESEARCH Pitfall 3). + if ( + self.status_actions + and Accessor('approval_status').resolve(record, quiet=True) == CampaignRun.ApprovalStatus.APPROVED + ): + decide_url = reverse('campaigns:decide', kwargs={'pk': record.pk}) + csrf_token = get_token(self.request) if self.request is not None else '' + # Buttons render for ANY APPROVED row regardless of current run_status (RESEARCH + # Open Question 1) -- a mis-click is correctable via the same UI and re-clicking + # is a harmless idempotent no-op; no revert button. + return format_html( + '
' + '' + '
' + '' + '' + '
', + decide_url, + csrf_token, + ) + return '' + decide_url = reverse('campaigns:decide', kwargs={'pk': record.pk}) + csrf_token = get_token(self.request) if self.request is not None else '' + if self.mode == 'resolve': + form_id = f'resolve-form-{record.pk}' + return format_html( + '
' + '' + '' + '
', + form_id, + decide_url, + csrf_token, + ) + form_id = f'decide-form-{record.pk}' + return format_html( + '
' + '' + '
' + '' + '' + '
', + form_id, + decide_url, + csrf_token, + record.pk, + ) diff --git a/solsys_code/campaign_urls.py b/solsys_code/campaign_urls.py new file mode 100644 index 00000000..5b0008d2 --- /dev/null +++ b/solsys_code/campaign_urls.py @@ -0,0 +1,34 @@ +"""FOMO campaigns URL conf -- the per-campaign table read path (VIEW-01/03/04). + +Mirrors solsys_code/calendar_urls.py's structure: app_name + a flat urlpatterns list. +""" + +from django.urls import path +from django.views.generic import TemplateView + +from solsys_code.campaign_views import ( + ApprovalQueueView, + CampaignGapAnalysisView, + CampaignListView, + CampaignRunDecisionView, + CampaignRunSubmissionView, + CampaignRunTableView, + SiteSearchView, +) + +app_name = 'campaigns' + +urlpatterns = [ + path('', CampaignListView.as_view(), name='list'), + path('submit/', CampaignRunSubmissionView.as_view(), name='submit'), + path( + 'submission-thanks/', + TemplateView.as_view(template_name='campaigns/submission_thanks.html'), + name='submission_thanks', + ), + path('approval-queue/', ApprovalQueueView.as_view(), name='approval_queue'), + path('site-search/', SiteSearchView.as_view(), name='site_search'), + path('/decide/', CampaignRunDecisionView.as_view(), name='decide'), + path('/gaps/', CampaignGapAnalysisView.as_view(), name='gap_analysis'), + path('/', CampaignRunTableView.as_view(), name='table'), +] diff --git a/solsys_code/campaign_utils.py b/solsys_code/campaign_utils.py new file mode 100644 index 00000000..1c956bdc --- /dev/null +++ b/solsys_code/campaign_utils.py @@ -0,0 +1,826 @@ +"""Shared helpers for the campaign-coordination CSV bootstrap import. + +Provides the D-08 3-tier site resolver, best-effort UT-time-window parsing, the +Observation-Status translation table, and the no-churn CampaignRun create-or-update +helper used by the ``import_campaign_csv`` management command. Mirrors +``calendar_utils.py``'s role for the three CalendarEvent sync commands: every function +here is structured as "never raise for expected messy data; return a usable value plus +an explicit flag" per the ``_derive_telescope_class`` precedent in ``calendar_utils.py``. +""" + +import difflib +import logging +import re +from datetime import date, datetime +from datetime import timezone as dt_timezone +from typing import Any + +import requests +from django.core.cache import cache +from django.db.utils import IntegrityError +from tom_dataservices.dataservices import MissingDataException + +from solsys_code.models import CampaignRun +from solsys_code.solsys_code_observatory.models import Observatory +from solsys_code.solsys_code_observatory.utils import MPCObscodeFetcher + +logger = logging.getLogger(__name__) + +# D-08 Pitfall 2: Observatory.obscode is CharField(max_length=4). Computed from the +# field itself (not hardcoded) so a future schema change can't silently desync this guard. +_MAX_OBSCODE_LEN = Observatory._meta.get_field('obscode').max_length + +# 22-06 gap closure: single source of truth for the tier-3 placeholder Observatory's name +# prefix. resolve_site()'s tier-3 fallback builds the name from this constant, and +# is_placeholder_observatory() below detects any Observatory carrying it -- so the two +# never drift out of sync (previously an ad-hoc string literal duplicated in each caller). +NEEDS_REVIEW_NAME_PREFIX = 'NEEDS REVIEW: ' + +# D-02/A2: the MPC obscode list changes far less often than gap-analysis results, so this +# mirrors campaign_gap.py's cache pattern (GAP_CACHE_TTL_SECONDS = 3600) with a much +# longer TTL. Single global pool -> a fixed cache key, no per-request parameters needed. +MPC_CANDIDATE_CACHE_TTL_SECONDS = 86400 # 24h +_MPC_CANDIDATE_CACHE_KEY = 'mpc_obscode_candidates' + +# When the MPC bulk fetch fails, build_site_candidates() still returns a usable local-only +# pool -- but it must NOT be cached for the full 24h TTL, or a single transient MPC blip +# poisons every site search for a whole day (the exact failure that shipped in Phase 22: +# a swallowed error degraded the pool and the degraded pool was then cached for 24h). Cache +# the degraded pool for only a short retry window so the next request re-attempts the MPC +# fetch, honoring build_site_candidates()'s documented "degrades gracefully" contract +# (graceful, not persistent, degradation). +MPC_CANDIDATE_FALLBACK_TTL_SECONDS = 60 # 1 min + +# UT Time Range formats confirmed against the real 3I/ATLAS sheet export (RESEARCH.md +# "Real 3I/ATLAS Sheet -- Verified Shape"): 'HH:MM - HH:MM' (tolerant of a ';' typo in +# place of ':'), a tilde-prefixed approximate hour ('~1 am', '~7:00:00 AM'), and a bare +# hour with an explicit 'UTC' marker ('5 UTC', '1 UTC'). Deliberately NOT a permissive +# general-purpose date/time parser (RESEARCH.md Anti-Patterns) -- each pattern requires +# an unambiguous marker (colon/semicolon range, leading '~', or trailing 'UTC') so a +# stray date-range or free-text garbage cell never "succeeds" into a wrong-but-plausible +# time. +_HHMM_RANGE = re.compile(r'(\d{1,2})[:;](\d{2})\s*(am|pm)?\s*-\s*(\d{1,2})[:;](\d{2})\s*(am|pm)?', re.IGNORECASE) +_APPROX_HOUR = re.compile(r'~\s*(\d{1,2})(?::\d{2})?(?::\d{2})?\s*(am|pm)?', re.IGNORECASE) +_BARE_HOUR_UTC = re.compile(r'(\d{1,2})\s*UTC\b', re.IGNORECASE) + +# D-12: full-date range, en-dash/em-dash/hyphen(s) or literal "to" separator. Anchored +# start-to-end (never .search()) so it never partially matches inside a longer garbage +# string -- Obs. Date is a structured column, not free prose (unlike the UT-time regexes +# above, which use .search() against genuinely free text). `-{1,2}` (260714-ilz) also +# accepts a double-hyphen separator ('2027-04-20 -- 2027-05-11') -- the ASCII-typable +# stand-in for an en/em dash that a public form submitter is far more likely to type than +# an actual Unicode dash character; the original CSV-sheet-derived single-hyphen/en-dash/ +# em-dash/"to" shapes are unaffected. +_DATE_RANGE_FULL = re.compile( + r'^(\d{4}-\d{2}-\d{2})\s*(?:to|-{1,2}|[–—])\s*(\d{4}-\d{2}-\d{2})$', + re.IGNORECASE, +) + +# D-11: compact same-month/rollover range, e.g. '2025-11-02 -25' or '2025-11-28 -05'. +# Second group is the day-of-month only (1-2 digits); rollover logic lives in the caller, +# not the regex, matching this module's existing convention (_HHMM_RANGE also keeps +# am/pm interpretation out of the pattern itself). +_DATE_RANGE_COMPACT = re.compile(r'^(\d{4})-(\d{2})-(\d{2})\s*-\s*(\d{1,2})$') + + +def _to_24h(hour: int, meridiem: str | None) -> int: + """Apply an optional am/pm marker to an hour parsed from a 12-hour-ish UT time cell. + + CR-01: the sheet's UT Time Range cells sometimes carry an explicit am/pm marker + (e.g. ``'~7:00:00 PM'``); the marker must be applied rather than silently discarded, + or a PM time parses as if it were AM (12 hours wrong, with no error and no flag). + + Args: + hour: the hour digits as parsed (1-12 for am/pm-marked input, 0-23 otherwise). + meridiem: ``'am'``/``'pm'`` (any case) if a marker was present, else ``None``. + + Returns: + int: the 24-hour-clock hour. + """ + if not meridiem: + return hour + meridiem = meridiem.lower() + if meridiem == 'am': + return 0 if hour == 12 else hour + return 12 if hour == 12 else hour + 12 + + +# Observation Status -> RunStatus translation (Pitfall 3): case-insensitive substring +# match, most-specific first, conservative REQUESTED default for anything unrecognized +# (a non-key field per D-05 -- an imprecise default must never block the row). +_STATUS_MAP = [ + ('cancel', CampaignRun.RunStatus.CANCELLED), + ('not awarded', CampaignRun.RunStatus.NOT_AWARDED), + ('weather', CampaignRun.RunStatus.WEATHER_TECH_FAILURE), + ('technical', CampaignRun.RunStatus.WEATHER_TECH_FAILURE), + ('publish', CampaignRun.RunStatus.PUBLISHED), + ('reduc', CampaignRun.RunStatus.REDUCED), + ('complet', CampaignRun.RunStatus.OBSERVED), + ('observ', CampaignRun.RunStatus.OBSERVED), + ('upcoming', CampaignRun.RunStatus.PLANNED), + ('planned', CampaignRun.RunStatus.PLANNED), +] + +# WR-08: a bare negation ("Not observed", no other keyword) contains the 'observ' +# substring but means the opposite -- map_observation_status skips the ('observ', ...) +# entry when this matches and no earlier, more-specific keyword already matched. +_NOT_OBSERVED_RE = re.compile(r'\bnot\s+observ', re.IGNORECASE) + + +def resolve_site(site_code_raw: str, *, create_placeholder: bool = True) -> tuple[Observatory | None, bool]: + """Resolve a raw Site Code string to an Observatory (D-08 3-tier resolution). + + Tier 1: match against an existing ``Observatory`` record. Tier 2: query the MPC + Obscodes API via ``MPCObscodeFetcher`` and create an ``Observatory`` row if found. + Tier 3: create a placeholder ``Observatory`` row, flagged for manual review -- unless + ``create_placeholder`` is False, in which case tier 3 is skipped entirely and the code + is flagged for manual review with no Observatory row created. A blank or oversized + (> ``Observatory.obscode``'s max length) code never reaches tier 1/2/3 at all -- it is + flagged immediately with no Observatory row created, so a code that can't possibly be + a real MPC obscode (e.g. JWST's 8-character spacecraft-style ``'500@-170'``) is never + truncated or fabricated (D-09/Pitfall 2). + + Args: + site_code_raw: the CSV row's raw ``Site Code`` cell value (may be blank, ``None``, + or contain leading/trailing whitespace). + create_placeholder: whether tier 3 may fabricate a placeholder ``Observatory`` row + when tiers 1 and 2 both miss. Defaults to ``True`` so the existing CSV-import + caller (already-vetted sheet data) is unaffected. Pass ``False`` for + unvetted/public free-text input (e.g. the campaign approval endpoint) so an + unresolvable site is flagged for manual review instead of fabricating a fake + Observatory row. + + Returns: + tuple[Observatory | None, bool]: ``(observatory_or_none, needs_review)``. Never + raises for expected messy-data cases. + """ + code = (site_code_raw or '').strip() + if not code: + return None, True # no code at all -- flag, no placeholder possible + + if len(code) > _MAX_OBSCODE_LEN: + # e.g. JWST's '500@-170' -- can't fit Observatory.obscode; don't fabricate a + # truncated/wrong site. Flag for manual review instead (Pitfall 2). + return None, True + + # Tier 1: existing Observatory record. CR-01 (22-REVIEW.md re-review): the matched row + # may itself be a tier-3 placeholder created by an earlier resolve_site() call for this + # same obscode (e.g. import_campaign_csv.py calling resolve_site() once per CSV row -- + # only the first row sharing a still-unresolved Site Code goes through tier 3; every + # later row hits this placeholder via tier 1). Derive needs_review from whether the + # matched row actually is a placeholder, never report False unconditionally, or a + # placeholder hit is silently mistaken for a genuine resolution. + try: + obs = Observatory.objects.get(obscode=code) + except Observatory.DoesNotExist: + pass + else: + return obs, is_placeholder_observatory(obs) + + # Tier 2: MPC Obscodes API (same call CreateObservatory.form_valid makes). The + # `errors` return value is intentionally unused beyond triggering the + # MissingDataException path below -- MPCObscodeFetcher.query() already logs the API + # error internally; don't double-log. + fetcher = MPCObscodeFetcher() + try: + fetcher.query(code, timeout=10) + except requests.exceptions.RequestException: + # WR-01: the MPC API call hung/failed at the network layer (timeout, DNS, + # connection reset, ...) -- import_campaign_csv calls resolve_site once per CSV + # row in a synchronous loop, so an unhandled network exception here would crash + # the whole batch import. Treat a network failure like an MPC miss and fall + # through to tier 3 rather than losing the rest of the import. + pass + else: + try: + # CR-01: to_observatory() never itself produces a placeholder-named row, but + # deriving needs_review the same way as every other success path keeps this + # tier consistent rather than special-cased. + obs = fetcher.to_observatory() + return obs, is_placeholder_observatory(obs) + except MissingDataException: + pass # no such obscode at MPC either -- fall through to tier 3 + except (KeyError, ValueError, TypeError): + # WR-04: to_observatory() reads several dict keys with no `.get()`/default, + # so a live MPC API response that's 200 OK but missing/malformed an expected + # key (KeyError) or has a value that fails a `float(...)`/similar conversion + # (ValueError/TypeError) would otherwise crash the whole import. Treat a + # malformed-but-"ok" response like an MPC miss and fall through to tier 3. + pass + except IntegrityError: + # Race: another row in this same import (or a concurrent process) already + # created it -- re-fetch instead of losing the row. CR-01: the racing writer + # could have been a concurrent tier-3 placeholder create for this same code, so + # the re-fetched row may itself be a placeholder -- derive needs_review from it + # rather than reporting False unconditionally. + try: + obs = Observatory.objects.get(obscode=code) + except Observatory.DoesNotExist: + # WR-02: Observatory.name is also unique=True, so an IntegrityError here + # isn't necessarily an obscode race -- it could be a name collision with + # a *different* obscode, in which case no Observatory exists for `code` + # and the re-fetch above would otherwise raise uncaught. Fall through to + # tier 3 instead of letting DoesNotExist propagate out of resolve_site. + pass + else: + return obs, is_placeholder_observatory(obs) + + if not create_placeholder: + # Public free-text submissions (unlike the already-vetted CSV import) should not + # auto-create a placeholder Observatory on approve -- flag for manual review only. + return None, True + + # Tier 3: placeholder, flagged for review (D-09 -- flag, don't silently guess). + try: + placeholder = Observatory.objects.create( + obscode=code, + name=f'{NEEDS_REVIEW_NAME_PREFIX}{code}', + short_name=code, + ) + except IntegrityError: + # WR-03: race protection matching tier 2's -- another row in this same import + # (or a concurrent process) already created an Observatory (placeholder or real) + # for this obscode. Re-fetch instead of crashing the import. + return Observatory.objects.get(obscode=code), True + return placeholder, True + + +def is_placeholder_observatory(observatory: Observatory | None) -> bool: + """Whether ``observatory`` is a tier-3 placeholder created by ``resolve_site()``. + + A pure, DB-free string check on the already-loaded ``name`` field -- callers pass + ``select_related('site')`` instances, so this must never trigger a query of its own. + + Args: + observatory: an ``Observatory`` instance, or ``None`` for an unresolved run. + + Returns: + bool: True when ``observatory`` is truthy and its ``name`` starts with + ``NEEDS_REVIEW_NAME_PREFIX``; False for ``None`` or a genuinely-resolved + Observatory. + """ + return bool(observatory) and observatory.name.startswith(NEEDS_REVIEW_NAME_PREFIX) + + +def _old_name_strings(old_names: Any) -> list[str]: + """Normalize an MPC record's ``old_names`` field into a list of candidate strings. + + The live MPC bulk obscodes API returns ``old_names`` as a JSON **list** of prior + names (confirmed live: 60 of 2,712 records, e.g. ``G96 -> ['Mt. Lemmon Survey']``), + or ``null`` for the vast majority. The single-record ``query()`` endpoint and older + fixtures have also been seen to carry a bare string. This helper collapses all three + shapes (list, str, None/missing) into a flat list of non-empty strings so callers can + treat every prior name as an independent fuzzy-match candidate. + + Passing a list straight through to the caller's ``candidate not in mapping`` / + ``mapping[candidate] = code`` dict operations previously raised + ``TypeError: unhashable type: 'list'`` (an unhashable list can be neither a dict key + nor a membership-test operand), which ``build_site_candidates()`` silently swallowed -- + discarding the entire MPC candidate pool. + + Args: + old_names: the record's raw ``old_names`` value -- a list of strings, a single + string, ``None``, or missing (any other type is treated as "no old names"). + + Returns: + list[str]: zero or more non-empty prior-name strings. Never raises. + """ + if isinstance(old_names, str): + return [old_names] if old_names else [] + if isinstance(old_names, list): + return [name for name in old_names if isinstance(name, str) and name] + return [] + + +def _candidate_str(value: Any) -> str: + """Coerce an MPC record field to a candidate string, treating any non-string as absent. + + Defence-in-depth against the bug #1 failure family (debug/site-search-mpc-no-match): the + live MPC bulk API has been observed to return an unexpectedly-shaped field -- ``old_names`` + arrives as a JSON **list** for 60/2,712 records. Passing a non-string straight into + ``_flatten_mpc_candidates()``'s dict-key / membership operations raises + ``TypeError: unhashable type: 'list'``, which ``build_site_candidates()``'s broad ``except`` + silently swallows -- discarding the ENTIRE MPC candidate pool for one malformed field. + ``old_names`` is normalized by ``_old_name_strings()``; this guards ``name_utf8`` / + ``short_name`` the same way, so a future shape surprise in ANY scalar candidate field + degrades to "skip that one field" rather than to a whole-pool drop. A live field-shape + audit (debug/site-search-degraded-pool-recurrence) confirmed all 2,712 records currently + carry str ``name_utf8``/``short_name`` -- this is purely forward-looking hardening and is a + no-op for every field shape seen in live data today (``str`` passes through unchanged, + ``None``/missing become ``''`` exactly as the previous ``... or ''`` did). + + Args: + value: the raw record field value -- a string, ``None``, or (defensively) any other type. + + Returns: + str: ``value`` when it is a string, else ``''``. Never raises. + """ + return value if isinstance(value, str) else '' + + +def _normalize_candidate(value: str) -> str: + """Collapse a candidate display string to its visible-rendered form. + + Runs of any whitespace are collapsed to a single space and leading/trailing whitespace + is stripped -- exactly the transformation a browser applies to a normal (non-``pre``) + HTML text node. This is the dedup key for the candidate pool: two byte-distinct strings + that differ only in whitespace runs (e.g. Z23's ``name_utf8`` ``'Nordic Optical + Telescope, La Palma'`` vs its ``old_names`` ``'Nordic Optical Telescope, La Palma'``) + render byte-for-byte identically in the suggestion dropdown, so treating them as distinct + candidates surfaces the SAME site twice (debug/duplicate-mpc-candidate-match). Live audit: + 32 of 2,712 MPC records carry an ``old_names`` whitespace-variant of their current name, + so this is a systemic dedup gap, not a Z23 one-off. Normalizing here is a visual no-op + (the browser already collapses the whitespace) that removes the redundant byte-variant + key at the pool's source, fixing every consumer -- substring match, difflib fuzzy match, + and ``selection_to_obscode()`` -- uniformly. + + Args: + value: a raw candidate display string (already coerced to ``str`` by callers). + + Returns: + str: the whitespace-normalized candidate. Never raises. + """ + return ' '.join(value.split()) + + +def _flatten_mpc_candidates(obscode_dict: dict) -> dict[str, str]: + """Flatten a bulk MPC obscodes dict into a fuzzy-matchable ``{string: obscode}`` map. + + Per RESEARCH.md Pattern 3 / Open Question 2: each record contributes its obscode, + ``name_utf8``, ``short_name``, and each of its ``old_names`` as candidate display + strings. First-seen wins on collision (rare -- distinct sites practically never share + a name); blank/falsy strings are skipped. ``old_names`` is normalized via + ``_old_name_strings()`` because the live bulk API returns it as a **list** (not a + string) -- each prior name becomes its own independent candidate rather than being + concatenated, so e.g. typing ``'Mt. Lemmon'`` matches ``G96``'s historical + ``'Mt. Lemmon Survey'`` name. ``name_utf8``/``short_name`` are coerced via + ``_candidate_str()`` and a non-dict record is skipped, so a single field- or record-shape + surprise in the live bulk data can never abort the whole flatten and silently drop all + ~2,712 candidates (the bug #1 failure family; debug/site-search-degraded-pool-recurrence). + + Args: + obscode_dict: dict keyed by 3-char obscode, as returned by + ``MPCObscodeFetcher.query_all()``. + + Returns: + dict[str, str]: candidate display string -> obscode. Never raises for expected + messy data (a missing/None/wrong-typed field is treated as absent and skipped). + """ + mapping: dict[str, str] = {} + for code, rec in obscode_dict.items(): + if not isinstance(rec, dict): + # A single malformed record must never abort the whole flatten -- that would + # discard every other record's candidates via build_site_candidates()'s broad + # except (the bug #1 whole-pool-drop failure mode). Skip it and keep the rest. + continue + candidates = [code, _candidate_str(rec.get('name_utf8')), _candidate_str(rec.get('short_name'))] + candidates.extend(_old_name_strings(rec.get('old_names'))) + for candidate in candidates: + # Normalize to the visible-rendered form BEFORE the first-seen dedup so two + # byte-distinct strings that render identically (e.g. name_utf8 vs an old_names + # whitespace-variant) collapse to one candidate rather than surfacing the same + # site twice (debug/duplicate-mpc-candidate-match). + candidate = _normalize_candidate(candidate) + if candidate and candidate not in mapping: + mapping[candidate] = code + return mapping + + +def _local_observatory_candidates() -> dict[str, str]: + """Build a ``{string: obscode}`` map from every local ``Observatory`` row. + + Candidate strings mirror ``_flatten_mpc_candidates()``'s field selection (obscode, + name, short_name, old_names) so the local and MPC-sourced pools merge uniformly. + First-seen wins on collision. + + CR-02 (22-REVIEW.md re-review): excludes tier-3 placeholder Observatories (created by + ``resolve_site()``'s ``create_placeholder`` fallback). Without this, a placeholder like + ``'NEEDS REVIEW: DCT'`` (obscode ``DCT``) would surface as a clickable suggestion in + both the public submission form's live search and the approval-queue/Sites-Needing- + Review correction widget -- clicking it maps back to the same placeholder's obscode and + (pre-CR-01-fix) resolve_site() would silently accept it as a genuine resolution. + + Returns: + dict[str, str]: candidate display string -> obscode. Never raises. + """ + mapping: dict[str, str] = {} + for obs in Observatory.objects.exclude(name__startswith=NEEDS_REVIEW_NAME_PREFIX): + for candidate in (obs.obscode, obs.name or '', obs.short_name or '', obs.old_names or ''): + # Same whitespace normalization as _flatten_mpc_candidates() so a local row whose + # name is a whitespace-variant of its MPC name doesn't reintroduce a visual + # duplicate when the two pools merge in build_site_candidates() + # (debug/duplicate-mpc-candidate-match). + candidate = _normalize_candidate(candidate) + if candidate and candidate not in mapping: + mapping[candidate] = obs.obscode + return mapping + + +def build_site_candidates() -> dict[str, str]: + """Build (and cache) the merged local + MPC fuzzy-match candidate pool (D-01/D-02). + + On a cache hit under ``'mpc_obscode_candidates'``, returns the cached pool without + re-fetching. On a miss, bulk-fetches the full MPC obscode list via + ``MPCObscodeFetcher.query_all()``, flattens it (``_flatten_mpc_candidates()``), merges + in every local ``Observatory`` row's candidate strings, caches the merged result for + ``MPC_CANDIDATE_CACHE_TTL_SECONDS``, and returns it. + + Mirrors ``resolve_site()``'s "never raise for expected messy data; return a usable + value plus an explicit flag" discipline (here: no explicit flag, since a network + failure degrades gracefully to a still-usable local-only pool rather than needing a + caller-visible error state): a bulk-fetch network/parse failure is caught narrowly and + falls back to the local-only ``Observatory`` pool, never raising into + ``ApprovalQueueView``'s page render (RESEARCH.md Environment Availability fallback). + + Returns: + dict[str, str]: candidate display string -> obscode, merged from the cached MPC + bulk list (or local-only on MPC failure) and every local ``Observatory`` row. + Never raises. + """ + cached = cache.get(_MPC_CANDIDATE_CACHE_KEY) + if cached is not None: + return cached + + mpc_candidates: dict[str, str] = {} + mpc_ok = False + try: + obscode_dict = MPCObscodeFetcher().query_all() + mpc_candidates = _flatten_mpc_candidates(obscode_dict) + mpc_ok = True + except (requests.exceptions.RequestException, ValueError, KeyError, TypeError, AttributeError): + # WR-style fallback (mirrors resolve_site()'s tier-2 network-failure handling): + # an MPC outage must never break the approval-queue page render -- fall through to + # a local-only pool below. WR-01: AttributeError is included because + # _flatten_mpc_candidates() calls .items()/.get() assuming obscode_dict (and each + # record) is a dict -- a bulk endpoint response that's drifted to a non-dict shape + # (e.g. a list or None) raises AttributeError, not one of the other caught types. + logger.debug('MPC bulk obscode fetch failed; falling back to local-only candidate pool.', exc_info=True) + + merged = dict(mpc_candidates) + # Local Observatory rows merge in last so an already-vetted local record's display + # string always wins a first-seen collision over the raw MPC bulk data. + for candidate, obscode in _local_observatory_candidates().items(): + merged.setdefault(candidate, obscode) + + # A full pool (MPC fetch succeeded) is cached for the long TTL; a degraded local-only + # pool (MPC fetch failed) is cached only briefly so the next request retries the MPC + # fetch instead of serving the degraded pool for a whole day. + ttl = MPC_CANDIDATE_CACHE_TTL_SECONDS if mpc_ok else MPC_CANDIDATE_FALLBACK_TTL_SECONDS + cache.set(_MPC_CANDIDATE_CACHE_KEY, merged, timeout=ttl) + return merged + + +# The site-search suggestion widget (site_search_results.html) writes the COMBINED display +# string `f'{display} ({obscode})'` into the site_selection input on click (e.g. +# 'Lowell Discovery Telescope (G37)'). selection_to_obscode() recovers the obscode from that +# trailing ' (obscode)' token. Greedy `.*` for the display part so a display that itself +# contains parentheses keeps everything up to the LAST group; `[^()]+` forbids nested parens +# in the obscode token so only a genuine trailing '(...)' is peeled off. +_SELECTION_DISPLAY_OBSCODE_RE = re.compile(r'^(?P.*) \((?P[^()]+)\)$') + + +def selection_to_obscode(selection: str) -> str: + """Map a site-search widget selection string back to its MPC obscode. + + The approval-queue site-search widgets submit whatever text is in the ``site_selection`` + input. When a staff member *clicks a suggestion*, the fragment's onclick handler + (``site_search_results.html``) sets that input to the COMBINED + ``f'{display} ({obscode})'`` string (e.g. ``'Lowell Discovery Telescope (G37)'``) -- + but ``build_site_candidates()``'s pool is keyed on the bare display strings and bare + obscodes only, never the combined form. A plain ``pool.get(selection)`` therefore MISSES + on every suggestion-selected value and the whole 30+ character combined string is passed + through to ``resolve_site()`` as a literal obscode, which rejects it as oversized + (``len > _MAX_OBSCODE_LEN``) and returns ``(None, True)`` -- surfacing as the generic + "Could not resolve that site" message (debug/site-resolve-list-old-names). This helper + resolves the selection robustly, in order: + + 1. Exact pool hit on the whole selection -- a bare obscode or bare display string + typed/selected verbatim -> its obscode. + 2. Otherwise, if the selection ends in a parenthesized token (the widget's + ``display (obscode)`` contract), recover that trailing token as the obscode, + preferring a pool hit on the leading display part when it is itself a candidate. + 3. Otherwise, return the selection unchanged (``resolve_site()`` will tier-1/2 it, + or reject it) -- backward-compatible with the previous + ``build_site_candidates().get(selection, selection)`` behavior. + + Args: + selection: the raw ``site_selection`` value (callers pass it already stripped). + + Returns: + str: the resolved obscode, or the original selection when it cannot be mapped. + """ + pool = build_site_candidates() + if selection in pool: + return pool[selection] + match = _SELECTION_DISPLAY_OBSCODE_RE.fullmatch(selection) + if match: + display, obscode = match.group('display'), match.group('obscode') + return pool.get(display, obscode) + return selection + + +def fuzzy_match_candidates(site_raw: str, candidate_pool: dict[str, str], n: int = 5) -> list[tuple[str, str]]: + """Fuzzy-match a raw submitted site string against a candidate pool (D-01/A3). + + Wraps ``difflib.get_close_matches`` (``cutoff=0.6`` -- difflib's own documented + default cutoff, per RESEARCH.md Assumption A3) and resolves each matched display + string back to its obscode via ``candidate_pool``. + + Args: + site_raw: the raw submitted/typed site text to match against the pool. Blank + input returns an empty list without invoking difflib. + candidate_pool: a ``{candidate_display_string: obscode}`` mapping, typically from + ``build_site_candidates()``. + n: maximum number of difflib matches to consider (default 5, difflib's own + documented default). Phase 22 P01: kept as an optional keyword-only-in-spirit + parameter (backward compatible default) so ``substring_or_fuzzy_match_candidates()``'s + fallback branch can request more than 5 without reimplementing the difflib + call; the existing single call site (``ApprovalQueueTable.render_site()``) is + unaffected by the default. + + Returns: + list[tuple[str, str]]: ranked ``(display_string, obscode)`` pairs, best match + first. Empty list when nothing clears the cutoff (e.g. an acronym/nickname + like ``'DCT'`` that difflib cannot bridge -- Pitfall 2). Never raises. + """ + text = (site_raw or '').strip() + if not text: + return [] + matches = difflib.get_close_matches(text, candidate_pool.keys(), n=n, cutoff=0.6) + return [(match, candidate_pool[match]) for match in matches] + + +def substring_or_fuzzy_match_candidates( + site_raw: str, candidate_pool: dict[str, str], *, limit: int = 8 +) -> list[tuple[str, str]]: + """Substring-first, difflib-fallback site match (D-04, Phase 22 P01). + + Case-insensitive containment over ``candidate_pool`` first -- this bridges short + partial queries like ``'Faulkes'`` against long official MPC strings (e.g. + ``'Faulkes Telescope South'``) that ``difflib.get_close_matches``'s whole-string + similarity scoring cannot reach at its 0.6 cutoff (Pitfall from 22-RESEARCH.md). + Falls back to ``fuzzy_match_candidates()`` for typo tolerance only when containment + finds nothing at all -- mirrors this module's "never raise for expected messy data" + discipline (``resolve_site()``/``build_site_candidates()``). + + Args: + site_raw: the raw submitted/typed live-search query text. Blank/whitespace-only + input returns an empty list without scanning the pool. + candidate_pool: a ``{candidate_display_string: obscode}`` mapping, typically from + ``build_site_candidates()``. + limit: maximum number of results to return (default 8, per CONTEXT.md's + "Claude's Discretion" suggestion count cap). + + Returns: + list[tuple[str, str]]: ranked ``(display_string, obscode)`` pairs. Substring hits + are sorted shortest/most-specific display string first; falls back to + ``fuzzy_match_candidates()``'s own ranking when there are no substring hits. + Never raises. + """ + text = (site_raw or '').strip() + if not text: + return [] + needle = text.lower() + hits = [(candidate, obscode) for candidate, obscode in candidate_pool.items() if needle in candidate.lower()] + if hits: + hits.sort(key=lambda pair: (len(pair[0]), pair[0])) # shortest/most-specific first + return hits[:limit] + return fuzzy_match_candidates(text, candidate_pool, n=limit)[:limit] + + +# D-02: 40 requests / 60s per IP -- within CONTEXT.md's 30-60/min guidance (Assumption +# A1). Exposed as a module-level constant (not a settings value) so tests can +# `patch.object(campaign_utils, 'SITE_SEARCH_THROTTLE_LIMIT', ...)` to a small number +# without touching Django settings. +SITE_SEARCH_THROTTLE_LIMIT = 40 +SITE_SEARCH_THROTTLE_WINDOW_SECONDS = 60 + + +def _check_and_increment_throttle(client_ip: str) -> bool: + """Fixed-window per-IP request throttle for the live-search endpoint (D-02). + + Uses only the already-imported ``django.core.cache.cache`` -- no new dependency + (D-02 explicitly forbids ``django-ratelimit``/DRF for this). ``cache.add()`` opens a + fresh window on the first request from an IP; subsequent requests within the window + increment the counter via ``cache.incr()``. + + Note: ``FileBasedCache``'s ``incr()`` is implemented as a plain get-then-set pair, not + atomic across processes (RESEARCH.md Pitfall 3) -- under concurrent requests from the + same IP this soft-throttle can under-count by a few, which is acceptable at this + project's single-dev-server deployment scale ("abuse protection", not a hard SLA). + ``REMOTE_ADDR`` is the only client-IP source used by the caller (Assumption A2) -- no + reverse-proxy/``X-Forwarded-For`` handling is configured in this project. Multiple + clients sharing one IP (e.g. behind the same reverse proxy) still share one bucket -- + that's the documented Assumption A2 limitation. WR-02 (22-REVIEW.md): the caller is + responsible for never passing a falsy ``client_ip`` here (e.g. a missing + ``REMOTE_ADDR``) -- doing so would silently collapse every IP-less client into one + ``site_search_throttle:`` bucket, which is a step beyond A2's known limitation. The + caller (``SiteSearchView.get()``) skips calling this function entirely when + ``REMOTE_ADDR`` is absent, rather than passing an empty string through. + + Args: + client_ip: the requesting client's IP address (typically ``request.META['REMOTE_ADDR']``); + must be truthy -- callers should skip throttling entirely rather than pass a falsy + value (see WR-02 note above). + + Returns: + bool: ``True`` if this request is within the per-window limit, ``False`` once the + IP has exceeded ``SITE_SEARCH_THROTTLE_LIMIT`` requests within + ``SITE_SEARCH_THROTTLE_WINDOW_SECONDS``. Never raises. + """ + key = f'site_search_throttle:{client_ip}' + added = cache.add(key, 1, timeout=SITE_SEARCH_THROTTLE_WINDOW_SECONDS) + if added: + return True + try: + count = cache.incr(key) + except ValueError: + # Key expired between add() and incr() (race) -- treat as a fresh window. + cache.set(key, 1, timeout=SITE_SEARCH_THROTTLE_WINDOW_SECONDS) + return True + return count <= SITE_SEARCH_THROTTLE_LIMIT + + +def parse_obs_window( + obs_date_raw: str, ut_range_raw: str +) -> tuple[date | None, date | None, str, bool, datetime | None, datetime | None, bool]: + """Best-effort parse of the sheet's Obs. Date + UT Time Range columns (D-11/D-12/D-13). + + Tries, in order, an exact ``YYYY-MM-DD`` date, a full-date range (``' to '``/en-dash/ + em-dash/hyphen separated), and a compact same-month/rollover range (``'2025-11-02 + -25'``). Anything that doesn't match any of those shapes -- including blank text, a + ``'YYYY-MM-?'`` marker, and free-text prose -- is a TBD row: ``window_start`` and + ``window_end`` are both ``None``, ``original_obs_date_raw`` carries the verbatim raw + text, and ``window_needs_review`` is ``True``. ``parse_obs_window()`` never raises for + any ``obs_date_raw`` input (D-13) -- this is a contract change from the previous + exact-date-or-raise behavior. + + ``ut_range_raw`` is only parsed for the single-night case (``window_start == + window_end``, both not ``None``); a range or TBD row skips UT parsing entirely (A1 -- + ``ut_start``/``ut_end``/``ut_needs_review`` are unused by every real caller, and a + multi-night window has no single night to anchor a UT time to). + + Args: + obs_date_raw: the CSV row's raw ``Obs. Date`` cell value -- an exact date, a + range (full-date or compact same-month/rollover), or unparseable free text. + ut_range_raw: the CSV row's raw ``UT Time Range`` cell value -- highly variable + free text in the real sheet (HH:MM ranges, semicolon typos, approximate + hours, bare-hour-plus-UTC shorthand, blank, or unparseable prose). + + Returns: + tuple[date | None, date | None, str, bool, datetime | None, datetime | None, bool]: + ``(window_start, window_end, original_obs_date_raw, window_needs_review, + ut_start, ut_end, ut_needs_review)``. ``ut_start``/``ut_end`` are tz-aware UTC + when present. Never raises. + """ + text = (obs_date_raw or '').strip() + + window_start: date | None = None + window_end: date | None = None + + try: + window_start = window_end = datetime.strptime(text, '%Y-%m-%d').date() + except ValueError: + match = _DATE_RANGE_FULL.match(text) + if match: + start_s, end_s = match.groups() + try: + window_start = datetime.strptime(start_s, '%Y-%m-%d').date() + window_end = datetime.strptime(end_s, '%Y-%m-%d').date() + if window_end < window_start: + # Reversed range (operands swapped, e.g. a source-sheet typo) -- + # treat like any other unparseable shape rather than silently + # accepting a window that claims zero dates (WR-01). + window_start = window_end = None + except ValueError: + window_start = window_end = None + else: + match = _DATE_RANGE_COMPACT.match(text) + if match: + year_s, month_s, day1_s, day2_s = match.groups() + year, month, day1, day2 = int(year_s), int(month_s), int(day1_s), int(day2_s) + try: + window_start = date(year, month, day1) + if day2 < day1: + # D-11 rollover: second number is smaller than the first + # day-of-month -- roll into the next month (and next year for a + # Dec -> Jan crossing). + window_end = date(year + 1, 1, day2) if month == 12 else date(year, month + 1, day2) + else: + window_end = date(year, month, day2) + except ValueError: + # e.g. day2=35 for a 28/29/30/31-day month, or day1 itself invalid -- + # stdlib date() already validates this; treat like any other + # unparseable shape and fall through to TBD. + window_start = window_end = None + + if window_start is None: + # No shape matched (blank, 'YYYY-MM-?', or genuine garbage) -- D-03/D-06/D-13 TBD + # state. No dedicated 'YYYY-MM-?' regex is needed: it falls through here naturally. + return None, None, text, True, None, None, False + + if window_start != window_end: + # Range row -- no single night to anchor a UT time to (A1); skip UT parsing. + return window_start, window_end, '', False, None, None, False + + # Single-night case (window_start == window_end): UT-Time-Range parsing unchanged. + obs_date = window_start + + match = _HHMM_RANGE.search(ut_range_raw or '') + if match: + h1_raw, m1, meridiem1, h2_raw, m2, meridiem2 = match.groups() + h1 = _to_24h(int(h1_raw), meridiem1) + h2 = _to_24h(int(h2_raw), meridiem2) + m1, m2 = int(m1), int(m2) + start = datetime(obs_date.year, obs_date.month, obs_date.day, h1, m1, tzinfo=dt_timezone.utc) + end = datetime(obs_date.year, obs_date.month, obs_date.day, h2, m2, tzinfo=dt_timezone.utc) + return window_start, window_end, '', False, start, end, False + + match = _APPROX_HOUR.search(ut_range_raw or '') + if match: + h = _to_24h(int(match.group(1)), match.group(2)) + start = datetime(obs_date.year, obs_date.month, obs_date.day, h, 0, tzinfo=dt_timezone.utc) + return window_start, window_end, '', False, start, None, False + + match = _BARE_HOUR_UTC.search(ut_range_raw or '') + if match: + h = int(match.group(1)) + start = datetime(obs_date.year, obs_date.month, obs_date.day, h, 0, tzinfo=dt_timezone.utc) + return window_start, window_end, '', False, start, None, False + + # Fallback: obs_date is valid but UT range isn't parseable at all (blank, garbage + # text, or a misplaced date-range) -- use midnight UTC (Pitfall 1), never skip here. + # Flagged via ut_needs_review=True (CR-02) since this fallback always resolves to the + # same timestamp for a given obs_date, so two distinct rows sharing telescope+date + # both falling back here would otherwise collide on the natural key. + start = datetime(obs_date.year, obs_date.month, obs_date.day, 0, 0, tzinfo=dt_timezone.utc) + return window_start, window_end, '', False, start, None, True + + +def map_observation_status(raw: str) -> str: + """Translate the sheet's free-text Observation Status into a RunStatus value (Pitfall 3). + + Case-insensitive substring match against a small, ordered translation table. Any + unrecognized string (including blank) falls back to the conservative + ``RunStatus.REQUESTED`` default rather than raising or guessing at a more specific + status -- ``run_status`` is a non-key field (D-05), so an imprecise default must + never block the row. A bare negation like ``'Not observed'`` (WR-08) is deliberately + *not* classified as ``OBSERVED`` even though it contains that substring -- unless a + more specific keyword also co-occurs (e.g. ``'Not observed -- weather'`` still maps to + ``WEATHER_TECH_FAILURE`` via the earlier, more-specific ``'weather'`` entry). + + Args: + raw: the CSV row's raw ``Observation Status`` cell value. + + Returns: + str: one of ``CampaignRun.RunStatus``'s values. Never raises. + """ + normalized = (raw or '').strip().lower() + for needle, status in _STATUS_MAP: + if needle == 'observ' and _NOT_OBSERVED_RE.search(normalized): + # WR-08: no more-specific keyword matched by this point (they're all earlier + # in the table), so this is a bare negation -- skip straight to REQUESTED + # rather than mis-classifying it as OBSERVED. + continue + if needle in normalized: + return status + return CampaignRun.RunStatus.REQUESTED + + +def insert_or_create_campaign_run(lookup: dict[str, Any], fields: dict[str, Any]) -> tuple[CampaignRun, str]: + """Create or update a CampaignRun, or leave it unchanged if no fields differ. + + Mirrors ``calendar_utils.insert_or_create_calendar_event()``'s no-churn + create-or-update contract: create a new ``CampaignRun`` if none exists for the given + lookup key (D-04's natural key), update it in place if any fields changed, or leave + it untouched if nothing changed (idempotent re-run, no spurious writes). + ``CampaignRun`` has no ``modified``/auto-now field, so an update issues + ``save(update_fields=list(fields))`` only -- unlike ``CalendarEvent``, there is no + timestamp field to include. + + Args: + lookup: keyword-argument mapping used as the unique lookup key for + ``CampaignRun.objects.get_or_create`` (D-04). Two shapes: resolved-window rows + key on (campaign, telescope_instrument, window_start, window_end); TBD rows key + on (campaign, telescope_instrument, contact_person, window_start__isnull=True) -- + the `window_start__isnull=True` guard is required so a TBD row never collides + with a resolved row sharing the same campaign/telescope/contact_person. + fields: field-value mapping of ``CampaignRun`` attributes to set when creating or + updating. Not merged with `lookup`; the caller is responsible for ensuring + the combined key+fields set is complete. + + Returns: + tuple[CampaignRun, str]: ``(run, action)`` where action is one of ``'created'`` + (new record written), ``'updated'`` (existing record changed and saved), or + ``'unchanged'`` (existing record matched all fields; no save issued). + """ + run, created = CampaignRun.objects.get_or_create(**lookup, defaults=fields) + if created: + return run, 'created' + changed = [f for f, v in fields.items() if getattr(run, f) != v] + if changed: + for f, v in fields.items(): + setattr(run, f, v) + run.save(update_fields=list(fields.keys())) + return run, 'updated' + return run, 'unchanged' diff --git a/solsys_code/campaign_views.py b/solsys_code/campaign_views.py new file mode 100644 index 00000000..62217023 --- /dev/null +++ b/solsys_code/campaign_views.py @@ -0,0 +1,951 @@ +"""Views for the per-campaign table read path (VIEW-01/02/03/04), the public submission write +path (SUBMIT-01/04/05), and the coverage-gap analysis view (GAP-02). + +Views: ``CampaignRunTableView`` (the sortable/paginated/filterable per-campaign table, +PII-gated at the queryset layer per D-13/VIEW-03), ``CampaignListView`` (D-03's campaigns +list page), ``CampaignRunSubmissionView`` (the public intake form), and +``CampaignGapAnalysisView`` (GET-triggered, cached, server-side-validated coverage-gap +analysis). Deliberately does not import ``solsys_code.views`` -- that module imports +``.ephem_utils`` at module load time, which triggers a ~1.6 GB SPICE kernel download (CLAUDE.md +"Heavy import side effect"). ``campaign_gap`` is safe to import at module scope here: it only +depends on ``telescope_runs.sun_event``, never the heavy SPICE-loading ephemeris module. +""" + +import logging +import re +from datetime import date, datetime +from datetime import time as dt_time +from datetime import timezone as dt_timezone + +from django.contrib import messages +from django.contrib.auth.models import User +from django.core.mail import send_mail +from django.db import IntegrityError, transaction +from django.db.models import Case, CharField, Count, EmailField, F, Value, When +from django.http import HttpResponse, HttpResponseBadRequest +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse, reverse_lazy +from django.views.generic import FormView, ListView, TemplateView, View +from django_filters.views import FilterView +from django_tables2 import RequestConfig +from django_tables2.views import SingleTableMixin +from tom_calendar.models import CalendarEvent +from tom_targets.models import TargetList + +from solsys_code.solsys_code_observatory.models import Observatory + +from .calendar_utils import insert_or_create_calendar_event +from .campaign_filters import CampaignRunFilterSet +from .campaign_forms import CampaignGapAnalysisForm, CampaignRunSubmissionForm +from .campaign_gap import clamp_date_range, get_or_compute_gap +from .campaign_tables import ApprovalQueueTable, CampaignRunTable +from .campaign_utils import ( + _check_and_increment_throttle, + build_site_candidates, + is_placeholder_observatory, + resolve_site, + selection_to_obscode, + substring_or_fuzzy_match_candidates, +) +from .mixins import StaffRequiredMixin +from .models import CampaignRun +from .telescope_runs import sun_event + +logger = logging.getLogger(__name__) + +# 22-REVIEWS.md finding 2: a conservative DOM-id allowlist for the `input_id` GET param +# echoed back into SiteSearchView's rendered fragment. HTML auto-escaping alone is NOT +# sufficient for a value embedded in the inline `onclick=` JS-string context -- browsers +# decode HTML entities before the JS parser runs, so an HTML-escaped quote still +# terminates the JS string. A non-matching value is replaced server-side with the +# default 'id_site_raw' before it ever reaches the template context (belt-and-suspenders +# on top of the template's own `|escapejs` filter). +_INPUT_ID_RE = re.compile(r'^[-A-Za-z0-9_:.]+$') + +# D-13/VIEW-03/T-15-01: the exact D-09 column list for non-staff requests. Deliberately +# enumerated explicitly (not introspected from CampaignRun._meta) so contact_person/ +# contact_email can never accidentally be included -- the SQL SELECT itself never fetches +# them for non-staff, per 15-RESEARCH.md Pitfall 1's "restrict the queryset, not just the +# rendered table" recommendation. +ALLOWED_FIELDS_FOR_NON_STAFF = [ + 'pk', + 'telescope_instrument', + 'site__short_name', + 'site_raw', + 'site_needs_review', + 'window_start', + 'window_end', + 'filters_bandpass', + 'run_status', + 'approval_status', + 'open_to_collaboration', + 'observation_details', + 'weather', + 'observation_outcome', + 'publication_plans', + 'comments', +] + + +class CampaignRunTableView(SingleTableMixin, FilterView): + """Sortable/paginated/filterable table of every CampaignRun for one campaign (VIEW-01/04). + + ``SingleTableMixin`` MUST be declared before ``FilterView`` in the class bases -- this MRO + order is load-bearing (15-RESEARCH.md Pitfall 4): it ensures ``FilterView.get()`` has + already set ``self.object_list = self.filterset.qs`` before ``SingleTableMixin`` builds the + table from it. Reversing the order can silently unfilter the table. + """ + + model = CampaignRun + table_class = CampaignRunTable + filterset_class = CampaignRunFilterSet + template_name = 'campaigns/campaignrun_table.html' + table_pagination = {'per_page': 25} # D-11 + + def get_queryset(self): + """Restrict to this campaign; non-staff get a PII-safe .values() queryset (D-13). + + D-04: default-sorts resolved rows first (most recent window_start first), TBD + (window_start is NULL) rows last -- portably across SQLite/PostgreSQL, which + default to opposite implicit NULL-ordering directions for DESC (RESEARCH.md + Pattern 4/Anti-Patterns). Applied here (not django-tables2's Meta.order_by, + which only compiles bare accessor strings) for both the staff and non-staff + branches. + """ + campaign_pk = self.kwargs['pk'] + qs = CampaignRun.objects.filter(campaign_id=campaign_pk) + if self.request.user.is_staff: + return qs.select_related('site').order_by(F('window_start').desc(nulls_last=True)) + # D-09/SUBMIT-02: non-staff see approved AND rejected runs; only pending_review is + # hidden. Queryset-level exclude (not a template conditional) so pending rows never + # enter the non-staff SELECT -- mirrors D-13's existing discipline (T-16-07). + qs = qs.exclude(approval_status=CampaignRun.ApprovalStatus.PENDING_REVIEW) + qs = qs.order_by(F('window_start').desc(nulls_last=True)) + # VIEW-05/T-21-02: gate contact_person/contact_email at the SQL SELECT via a per-row + # Case/When annotation keyed on the submitter's own opt-in flag -- an opted-out row's + # real contact values are never fetched, only an empty string. ALLOWED_FIELDS_FOR_NON_STAFF + # itself deliberately does NOT list contact_person/contact_email (RESEARCH.md + # Anti-Pattern); they arrive only via this annotation. + # + # .values() MUST be called before .annotate() here (not after, despite that reading + # more naturally): Django's annotate() rejects an alias that collides with a real model + # field name ("The annotation 'contact_person' conflicts with a field on the model"), + # and that check is against the model's full field list unless .values() has already + # narrowed QuerySet._fields -- calling .values() first (without contact_person/ + # contact_email in the field list) makes the alias check pass. contact_public_opt_in + # itself doesn't need to be in the .values() field list for the When() condition below + # to reference it -- Django resolves F()/condition expressions against the underlying + # column regardless of the projected .values() field list. + qs = qs.values(*[f for f in ALLOWED_FIELDS_FOR_NON_STAFF if f not in ('contact_person', 'contact_email')]) + return qs.annotate( + contact_person=Case( + When(contact_public_opt_in=True, then=F('contact_person')), + default=Value(''), + output_field=CharField(), + ), + contact_email=Case( + When(contact_public_opt_in=True, then=F('contact_email')), + default=Value(''), + output_field=EmailField(), + ), + ) + + def get_table_kwargs(self): + """D-04: 'order_by': () suppresses django-tables2's own default sort so it doesn't + clobber get_queryset()'s nulls-last ordering (mirrors the existing + decided_table = ApprovalQueueTable(..., order_by=()) precedent in + ApprovalQueueView below). Interactive column-header sorting (RequestConfig) + still works normally on top of this. + + VIEW-05: contact_person/contact_email are no longer excluded for non-staff -- they're + always safe to render now (blank string for opted-out rows, populated for opted-in + ones), gated at the SQL SELECT by get_queryset()'s Case/When annotation, not here. + """ + return {'order_by': ()} + + def get_context_data(self, **kwargs): + """Add the campaign (TargetList) and D-14 gap-analysis-button availability to context.""" + context = super().get_context_data(**kwargs) + context['campaign'] = get_object_or_404(TargetList, pk=self.kwargs['pk']) + # D-14: reuse gap_analysis_available() (defined below) rather than duplicating its + # target-count / resolved-site logic here -- gates the "Show Coverage Gaps" button. + context['gap_analysis_available'] = gap_analysis_available(context['campaign']) + return context + + +class CampaignListView(ListView): + """Lists every TargetList that has >= 1 CampaignRun, each linking to its table (D-03). + + ``TargetList`` has no "is this a campaign" flag -- "campaign" is purely operational: a + TargetList with campaign_runs__isnull=False (15-RESEARCH.md Pitfall 3). Never uses + TargetList.objects.all(), which would include unrelated saved searches/groupings. + """ + + queryset = ( + TargetList.objects.filter(campaign_runs__isnull=False).distinct().annotate(run_count=Count('campaign_runs')) + ) + template_name = 'campaigns/campaign_list.html' + context_object_name = 'campaigns' + + def get_context_data(self, **kwargs): + """Add pending_count for the staff-only "N pending review" banner (D-01). + + Computed unconditionally -- the template gates its display on request.user.is_staff, + so it's harmless to compute for anonymous/non-staff visitors too (D-10: list + membership itself is unchanged). + """ + context = super().get_context_data(**kwargs) + context['pending_count'] = CampaignRun.objects.filter( + approval_status=CampaignRun.ApprovalStatus.PENDING_REVIEW + ).count() + return context + + +class CampaignRunSubmissionView(FormView): + """Public intake form for a single observing run, pending staff review (SUBMIT-01/04/05). + + A honeypot trip (``alt_contact_info`` populated) short-circuits to the exact same + thanks-page redirect as a genuine submission -- no ``CampaignRun`` created, no email sent, + no error signal (SUBMIT-04, 16-RESEARCH.md Pattern 3). A natural-key collision on + ``.objects.create()`` (Pitfall 4) degrades to a friendly non-field form error, never a 500. + """ + + form_class = CampaignRunSubmissionForm + template_name = 'campaigns/campaignrun_submit_form.html' + success_url = reverse_lazy('campaigns:submission_thanks') + + def form_valid(self, form): + """Create the CampaignRun (or silently drop a honeypot trip) and notify staff.""" + if form.cleaned_data.get('alt_contact_info'): + # SUBMIT-04: bot tripped the honeypot -- fall straight through to the same success + # redirect as a genuine submission. No create, no email, no error signal. + return redirect('campaigns:submission_thanks') + try: + # Wrapped in its own atomic block (savepoint): without it, the IntegrityError + # caught below poisons the outer request/test transaction, and any subsequent + # query (e.g. re-rendering the form's ModelChoiceField) raises + # TransactionManagementError instead of the intended friendly form error. + with transaction.atomic(): + run = CampaignRun.objects.create( + campaign=form.cleaned_data['campaign'], + telescope_instrument=form.cleaned_data['telescope_instrument'], + site_raw=form.cleaned_data['site_raw'], + # SCHED-02: window_start/window_end come from the form's clean(), which + # runs the free-text obs_date through parse_obs_window() -- single-night + # collapse (start == end) for one date or an equal-endpoint range, a real + # start..end span for a multi-night range, and both None for a blank + # (TBD) submission. + window_start=form.cleaned_data['window_start'], + window_end=form.cleaned_data['window_end'], + filters_bandpass=form.cleaned_data['filters_bandpass'], + observation_details=form.cleaned_data['observation_details'], + open_to_collaboration=form.cleaned_data['open_to_collaboration'], + contact_person=form.cleaned_data['contact_person'], + contact_email=form.cleaned_data['contact_email'], + contact_public_opt_in=form.cleaned_data['contact_public_opt_in'], + comments=form.cleaned_data['comments'], + # approval_status intentionally not set -- model default is PENDING_REVIEW. + # site/site_needs_review intentionally not set -- resolved at approval + # time (D-07). + ) + except IntegrityError: + # Pitfall 4: two submitters proposing the same campaign+telescope_instrument+ + # resolved window (a single night OR an identical range) -- or the same + # campaign+telescope_instrument+contact_person when the date is left blank + # (TBD) -- collide on one of CampaignRun's two partial natural-key + # UniqueConstraints. Friendly form error, never a 500. Requirement 7 (see + # 260714-ilz-SUMMARY.md): this handler already covers the range case unchanged; + # only the wording below was broadened to read correctly for a window as well + # as a single date. + form.add_error( + None, + 'A run for this telescope for this observing window already exists for this campaign. ' + 'Check the campaign table, or contact a coordinator if you believe this is a mistake.', + ) + return self.form_invalid(form) + self._notify_staff(run) + return redirect('campaigns:submission_thanks') + + def _notify_staff(self, run): + """Email every staff user with an email on file that a submission is pending (SUBMIT-05). + + Body/subject intentionally carry no PII (D-04) -- a bare ping plus the approval-queue + link, nothing about the submitter, telescope, or campaign. + """ + recipients = list(User.objects.filter(is_staff=True).exclude(email='').values_list('email', flat=True)) + if not recipients: + return # no staff with an email on file -- nothing to notify, not an error + # WR-03: campaigns:approval_queue is wired up by campaign_urls.py/src/fomo/urls.py in + # this same shipped changeset, so reverse() always succeeds here -- no NoReverseMatch + # fallback needed. + queue_url = self.request.build_absolute_uri(reverse('campaigns:approval_queue')) + send_mail( + subject='FOMO: new campaign run submission pending review', + message=f'A new run submission is pending review: {queue_url}', + from_email=None, + recipient_list=recipients, + fail_silently=True, # Pitfall 6: a mail outage must never break the submission + ) + + +class ApprovalQueueView(StaffRequiredMixin, TemplateView): + """Staff-only two-section approval queue: pending review + recently decided (D-01/D-02). + + Two independent ``ApprovalQueueTable`` instances are built by hand from two separate + querysets (16-RESEARCH.md Pattern 5) rather than routed through ``MultiTableMixin``, since + the pending/decided querysets have genuinely asymmetric filtering (not a list of symmetric + tables). ``StaffRequiredMixin`` gates the whole view -- this page must NOT follow Phase 15's + soft-filter (``.values()``) pattern; anonymous/non-staff requests are redirected before any + pending-submission content (which includes contact PII) is ever rendered (T-16-03). + """ + + template_name = 'campaigns/approval_queue.html' + + def get_context_data(self, **kwargs): + """Build the pending (actionable) and recently-decided (read-only) tables.""" + context = super().get_context_data(**kwargs) + pending_qs = CampaignRun.objects.filter( + approval_status=CampaignRun.ApprovalStatus.PENDING_REVIEW + ).select_related('campaign', 'site') + # Pitfall 1: CampaignRun has no modified/timestamp field -- order by -pk (a reasonable + # recency proxy) and cap at 20 rows. Materialized to a list before handing it to the + # table: django-tables2's table construction would otherwise re-sort the data, and + # Django refuses to call .order_by() again on an already-sliced queryset (`Cannot + # reorder a query once a slice has been taken`). A plain list sidesteps that entirely + # (django-tables2 sorts lists in Python via TableListData.order_by), and order_by=() + # below suppresses any default sort so the -pk selection order is preserved on first + # render (D-04's nulls-last window sort is a CampaignRunTableView-only concern; this + # queue view intentionally orders by recency, not window_start). + decided_qs = ( + CampaignRun.objects.exclude(approval_status=CampaignRun.ApprovalStatus.PENDING_REVIEW) + .select_related('campaign', 'site') + .order_by('-pk')[:20] + ) + # SITE-01/Pitfall 5: build the merged local+MPC candidate pool exactly once per + # request (never per row) -- build_site_candidates() is itself 24h-cached, but + # calling it once here still avoids a per-row cache.get() round-trip. Never + # raises (Plan 21-01's local-only fallback), so no try/except is needed here. + candidate_pool = build_site_candidates() + pending_table = ApprovalQueueTable( + pending_qs, + prefix='pending-', + request=self.request, + candidate_pool=candidate_pool, + empty_text='No submissions waiting for review.', + ) + decided_table = ApprovalQueueTable( + list(decided_qs), + prefix='decided-', + show_actions=False, + status_actions=True, + request=self.request, + empty_text='No decisions recorded yet.', + order_by=(), + ) + # D-07: approved runs whose site never resolved -- the "dead end" this phase closes. + # Deliberately NO row cap (unlike decided_qs's [:20] audit-log cap): this is a live + # work queue of items genuinely needing staff action, and capping it would hide + # actionable rows. Naturally includes the projection-failed retry state (site set, + # flag still True) since the filter is on site_needs_review alone. + review_qs = ( + CampaignRun.objects.filter(approval_status=CampaignRun.ApprovalStatus.APPROVED, site_needs_review=True) + .select_related('campaign', 'site') + .order_by('-pk') + ) + review_table = ApprovalQueueTable( + list(review_qs), + prefix='review-', + request=self.request, + # Pitfall 5: reuse the SAME candidate_pool already computed above for + # pending_table -- never call build_site_candidates() a second time per request. + candidate_pool=candidate_pool, + mode='resolve', + empty_text='No sites currently need review.', + order_by=(), + ) + RequestConfig(self.request).configure(pending_table) + RequestConfig(self.request).configure(decided_table) + RequestConfig(self.request).configure(review_table) + context['pending_table'] = pending_table + context['decided_table'] = decided_table + context['review_table'] = review_table + return context + + +# D-03: two distinct title prefixes for the two terminal run_status outcomes staff can set +# from the Decided table. Keyed on the RunStatus enum member (never derived from raw request +# text -- V5 Input Validation); must stay byte-identical to the '[WEATHERED]' string appended +# to calendar_display_extras._TERMINAL_PREFIXES (Task 3) so the box-shadow ring applies. +_RUN_STATUS_CALENDAR_PREFIX = { + CampaignRun.RunStatus.CANCELLED: '[CANCELLED]', + CampaignRun.RunStatus.WEATHER_TECH_FAILURE: '[WEATHERED]', +} + +# T-23-05: fixed whitelist mapping a POST action value to the RunStatus it sets -- the +# run_status value written is always looked up here, never taken from raw request text. +_ACTION_TO_RUN_STATUS = { + 'mark_cancelled': CampaignRun.RunStatus.CANCELLED, + 'mark_weather_failure': CampaignRun.RunStatus.WEATHER_TECH_FAILURE, +} + + +def _project_calendar_event(run: CampaignRun) -> bool: + """CAL-01/CAL-02 CalendarEvent projection (D-08), extracted from the approve branch. + + Returns True when ``insert_or_create_calendar_event()`` was actually called (an event was + created/updated), False when projection was skipped by design (range/TBD run, or missing + telescope_instrument/site) -- 22-REVIEWS.md finding 6: this bool drives the resolve_site + action's two distinct success messages. RAISES ValueError when ``sun_event()`` fails (e.g. + a Tier-2-resolved site with a blank ``timezone`` -- CR-01), and MAY RAISE on any other + unexpected failure (e.g. ``insert_or_create_calendar_event()`` itself failing) -- this + helper does NO error-handling of its own for genuine failures; callers own + revert-vs-non-revert behavior. ``resolve_site()`` must treat any raise here as "projection + attempted but failed" (keep ``site_needs_review=True``, warn instead of claiming success); + ``approve()`` has no retry surface to protect and instead catches-and-swallows the + ValueError case specifically at its call site to preserve its original behavior (approval + still succeeds even when the calendar entry couldn't be projected). + """ + # D-06/CAL-01: CalendarEvent.start_time/end_time are non-nullable -- only project a + # single concrete night (window_start == window_end); a resolved site is required to + # pick the ground-vs-space branch. A range, TBD run, or unresolved site simply doesn't + # get a CalendarEvent yet. + if not (run.telescope_instrument and run.site and run.window_start and run.window_start == run.window_end): + return False + event_fields = { + 'title': f'{run.campaign.name}: {run.telescope_instrument}', + 'description': run.observation_details, + 'target_list': run.campaign, # CAL-02 + 'telescope': run.telescope_instrument, + } + if run.site.observations_type == Observatory.SATELLITE_OBSTYPE: + # Space-based observatory: no fixed horizon for sun_event() to work against -- use + # a midnight-UTC placeholder spanning the window date. + event_fields['start_time'] = datetime.combine(run.window_start, dt_time(0, 0), tzinfo=dt_timezone.utc) + event_fields['end_time'] = datetime.combine(run.window_end, dt_time(23, 59), tzinfo=dt_timezone.utc) + # Never construct CalendarEvent directly -- always route through the shared helper + # (Don't Hand-Roll) so the CAMPAIGN: namespace stays collision-safe against the + # LCO/Gemini/classical sync commands (T-16-09). + insert_or_create_calendar_event({'url': f'CAMPAIGN:{run.pk}'}, fields=event_fields) + return True + # Ground-based observatory: reuse the same dip-corrected sunset/sunrise convention the + # rest of the calendar feature already uses (kind='sun', not 'dark' -- Pitfall 6). A + # ValueError (e.g. blank site.timezone, or no 2 sun-altitude crossings) is logged and + # re-raised (CR-01) -- callers decide whether that's a by-design skip (approve()) or a + # real failure that must keep the retry surface open (resolve_site()). + # + # IN-02 (19-REVIEW.md): this branch also catches OCCULTATION_OBSTYPE and RADAR_OBSTYPE + # sites, not just OPTICAL_OBSTYPE -- every non-SATELLITE Observatory.OBSTYPE_CHOICES + # member unconditionally gets the dip-corrected dark-window treatment. That's a + # deliberate simplification for this milestone; scope this to Observatory.OPTICAL_OBSTYPE + # explicitly, with OCCULTATION/RADAR falling back to no projection, when those site types + # get real support. + try: + sunset, sunrise = sun_event(run.site, run.window_start, kind='sun') + except ValueError: + logger.debug( + 'sun_event(sun) raised for site=%s date=%s; re-raising so callers that need the ' + 'retry guarantee (resolve_site) see this as a failure, not a by-design skip.', + run.site, + run.window_start, + ) + raise # CR-01: never silently swallow this -- see docstring above. + event_fields['start_time'] = sunset.to_datetime(timezone=dt_timezone.utc).replace(microsecond=0) + event_fields['end_time'] = sunrise.to_datetime(timezone=dt_timezone.utc).replace(microsecond=0) + insert_or_create_calendar_event({'url': f'CAMPAIGN:{run.pk}'}, fields=event_fields) + return True + + +class CampaignRunDecisionView(StaffRequiredMixin, View): + """POST-only atomic approve/reject decision endpoint (SUBMIT-03) + calendar projection, + plus the resolve_site action (D-08) that resolves an approved run's still-unmatched site + and retroactively projects the calendar event approval skipped. + + A single conditional ``.filter(pk=pk, approval_status=PENDING_REVIEW).update(...)`` proves + the double-approve no-op (T-16-02): a second decision POST on an already-decided row + matches zero rows, so the calendar projection below is never re-triggered (CAL-03). + ``http_method_names = ['post']`` ensures a GET (crawler prefetch, bare ````) can + never trigger a state change (T-16-06). + """ + + http_method_names = ['post'] + + def post(self, request, pk): + """Atomically transition a CampaignRun and, on approve, project a CalendarEvent.""" + action = request.POST.get('action') + if action not in ('approve', 'reject', 'resolve_site', 'mark_cancelled', 'mark_weather_failure'): + return HttpResponseBadRequest() + if action == 'resolve_site': + return self._resolve_site(request, pk) + if action in ('mark_cancelled', 'mark_weather_failure'): + return self._set_run_status(request, pk, action) + new_status = CampaignRun.ApprovalStatus.APPROVED if action == 'approve' else CampaignRun.ApprovalStatus.REJECTED + updated_count = CampaignRun.objects.filter( + pk=pk, approval_status=CampaignRun.ApprovalStatus.PENDING_REVIEW + ).update(approval_status=new_status) + + if updated_count == 1 and action == 'approve': + try: + run = CampaignRun.objects.get(pk=pk) + # D-06: only resolve the site once. An already-resolved run.site (from CSV + # import, tier 1/2 auto-resolution, or a prior staff-UI resolution) must never + # be re-resolved on a later approve -- e.g. after the except Exception revert + # below reverts approval_status back to PENDING_REVIEW while leaving run.site + # set, a second approve POST would otherwise re-hit resolve_site() + # unconditionally (RESEARCH.md Pitfall 3, the live clobbering bug this closes). + # A satellite-type site_selection (250/274/289) still falls through to + # (None, True) via resolve_site()'s to_observatory() TypeError path -- expected, + # pre-existing behavior, not a Phase 21 regression (RESEARCH.md Pitfall 4). + # WR-01 (22-REVIEW.md re-review): mirrors _resolve_site()'s placeholder-aware + # guard below -- a run whose site is already a tier-3 placeholder (e.g. from + # CSV import) is not a genuine resolution either, so it must still re-enter + # resolution here, not only when site is None. + if run.site is None or is_placeholder_observatory(run.site): + # D-07: reuse the existing 3-tier site resolver rather than + # re-implementing it. SITE-02: prefer the staff-submitted site_selection + # (Plan 21-03's inline input) over the originally-submitted site_raw, + # falling back to site_raw when blank. On approve we resolve the site but + # never auto-create a placeholder Observatory for unresolvable public free + # text (unlike the already-vetted CSV import path) -- the run is still + # approved with site=None + site_needs_review=True (site failure never + # blocks approval; the calendar projection below needs a resolved site, so + # an unresolved site simply means no CalendarEvent yet, not a blocked + # approval). + selection = request.POST.get('site_selection', '').strip() or run.site_raw + # CR-01: the live-search widget offered on the row + # (ApprovalQueueTable.render_site -> _render_site_search_widget) surfaces + # MPC-sourced display strings (name_utf8/short_name/old_names), not + # obscodes -- resolve the submitted text back to its obscode before calling + # resolve_site(), which otherwise treats its argument as a literal obscode. + # selection_to_obscode() maps an exact pool hit (a display string or + # obscode picked/typed verbatim) AND the suggestion widget's COMBINED + # 'display (obscode)' click value (debug/site-resolve-list-old-names) back + # to the obscode; anything else passes through unchanged. + obscode_selection = selection_to_obscode(selection) + site, needs_review = resolve_site(obscode_selection, create_placeholder=False) + run.site, run.site_needs_review = site, needs_review + run.save(update_fields=['site', 'site_needs_review']) + + # Projection extracted into the shared _project_calendar_event() helper + # (22-REVIEWS.md finding 6); the approve branch ignores its bool return. + # CR-01: _project_calendar_event() now raises ValueError when sun_event() + # fails (e.g. a Tier-2-resolved site with a blank timezone) so resolve_site() + # can treat it as a real failure. approve() has no retry surface to protect + # (unlike resolve_site()'s "Sites Needing Review" row), so it swallows + # specifically this expected-failure-mode ValueError here to preserve its + # original behavior: the approval still succeeds without a CalendarEvent. + # Anything else _project_calendar_event() raises (e.g. + # insert_or_create_calendar_event() itself failing) is a genuine unexpected + # failure and still falls through to the broader except Exception below, + # which reverts the approval. + try: + _project_calendar_event(run) + except ValueError: + logger.debug( + 'Calendar projection skipped for CampaignRun %s on approve ' + '(sun_event ValueError, e.g. blank site timezone).', + pk, + ) + except Exception: + # CR-01: the conditional .update() above is its own auto-committed statement, + # so the APPROVED transition has already landed. If site resolution (a network + # call to the MPC Obscodes API) or calendar projection then fails, revert + # approval_status back to PENDING_REVIEW so the run is never left permanently + # "approved" with no CalendarEvent and no way to re-decide it -- without this, + # the double-approve guard above makes that half-approved state unrecoverable + # through the UI. + logger.exception('Approve side-effects failed for CampaignRun %s; reverted to pending review.', pk) + CampaignRun.objects.filter(pk=pk).update(approval_status=CampaignRun.ApprovalStatus.PENDING_REVIEW) + messages.error( + request, + 'Approval failed while resolving the site or projecting the calendar event. ' + 'This run has been reset to pending review -- please try again.', + ) + return redirect('campaigns:approval_queue') + messages.success(request, 'Run approved.') + elif updated_count == 1: + messages.success(request, 'Run rejected.') + elif CampaignRun.objects.filter(pk=pk).exists(): + # WR-01: the conditional .update() above returns 0 both when the row exists but + # was already decided, and when pk never existed at all -- distinguish the two so a + # deleted/stale/tampered pk gets an honest "no longer exists" message instead of the + # factually-wrong "already decided by someone else". + messages.warning(request, 'This run was already decided by someone else.') + else: + messages.error(request, 'This run no longer exists.') + return redirect('campaigns:approval_queue') + + def _resolve_site(self, request, pk): + """D-07/D-08: resolve an approved run's still-unmatched site, then retroactively + project the CalendarEvent approval skipped. + + Ordering is deliberately load-bearing (22-REVIEWS.md findings 3/5/6/8c): + ``site_needs_review`` is cleared ONLY after ``_project_calendar_event()`` returns + without raising -- never before, and never on a projection failure -- so a failed + projection leaves the run visible in the Sites Needing Review table (its retry + surface) instead of vanishing into a dead end. The site write itself is a single + conditional queryset update (not a plain re-fetch + in-Python check) so two racing + staff POSTs cannot both claim the write. + + 22-06 gap closure (UAT gap 2B): a tier-3 PLACEHOLDER site (``resolve_site()``'s + ``create_placeholder`` fallback -- name prefixed ``NEEDS REVIEW: ``) is not a + genuine resolution, so it's also eligible for replacement here, alongside the + site=None case -- see ``is_placeholder_observatory()`` below. A genuinely-resolved + (non-placeholder) site is still never re-resolved (D-06). + """ + # Pitfall 2: re-fetch fresh from the DB -- never trust a stale in-memory instance. + run = get_object_or_404(CampaignRun, pk=pk) + + # Business-logic bypass guard (Security "business-logic bypass" domain): validate + # state server-side, never just trust the button was only offered on eligible rows. + if run.approval_status != CampaignRun.ApprovalStatus.APPROVED or not run.site_needs_review: + messages.warning(request, 'This run is not awaiting site resolution.') + return redirect('campaigns:approval_queue') + + # 22-06: capture the pre-read site pk (None when unresolved, the placeholder + # Observatory's own pk when a placeholder) BEFORE any write -- the conditional + # claim below keys on this exact value so two racing staff POSTs can never + # double-write (D-06). + previous_site_id = run.site_id + + # D-06 never-re-resolve guard, extended for 22-06: only resolve when the site + # isn't set yet, OR is a tier-3 placeholder (not a genuine resolution). A run with + # a REAL Observatory already set + site_needs_review still True is the + # projection-failed retry state (finding 8c) -- resolve_site is never called again + # for it; it falls straight through to the projection retry below. + if run.site is None or is_placeholder_observatory(run.site): + # SITE-02: prefer the staff-submitted site_selection over the originally- + # submitted site_raw, falling back to site_raw when blank. + selection = request.POST.get('site_selection', '').strip() or run.site_raw + # Map the widget selection back to its obscode before calling resolve_site(), + # which otherwise treats its argument as a literal obscode. selection_to_obscode() + # handles the suggestion widget's COMBINED 'display (obscode)' input value (the + # exact 'Could not resolve that site' bug in debug/site-resolve-list-old-names) as + # well as a bare obscode / bare display string typed or picked verbatim. + obscode_selection = selection_to_obscode(selection) + site, needs_review = resolve_site(obscode_selection, create_placeholder=False) + if site is None: + # Nothing was written -- D-09: never fabricate a second placeholder from + # unresolvable input. The flag is already True, the row (still pointing at + # its existing placeholder, if any) stays in the review table for another + # attempt. + messages.error( + request, + 'Could not resolve that site. Try a different search term or an exact ' + 'MPC code, or use Create new Observatory.', + ) + return redirect('campaigns:approval_queue') + + # 22-REVIEWS.md finding 5: claim the site write with a single conditional + # queryset update mirroring the approve/reject staleness guard + # (`updated_count == 1` discipline) -- deliberately writing `site` ONLY, never + # `site_needs_review` (finding 3: the flag must never clear before a successful + # projection). Not using transaction.atomic()+select_for_update() here -- the + # conditional-update claim is this codebase's established guard and behaves + # uniformly on SQLite. + # + # 22-06: keyed on `site_id=previous_site_id` (not the old hard-coded + # `site__isnull=True`) so the same conditional-claim guard covers both the + # unresolved case (Django treats `site_id=None` as IS NULL -- byte-equivalent + # to the old filter) and the placeholder-replacement case: a competing POST + # that already changed the site away from `previous_site_id` matches zero rows. + claimed = CampaignRun.objects.filter( + pk=pk, + approval_status=CampaignRun.ApprovalStatus.APPROVED, + site_needs_review=True, + site_id=previous_site_id, + ).update(site=site) + if claimed == 0: + # A racing staff POST resolved (or is resolving) this run first -- the + # loser's site value is never written, and no projection fires for it. + messages.warning(request, "This run's site was already resolved by someone else.") + return redirect('campaigns:approval_queue') + run.refresh_from_db() + + # WR-03 (22-REVIEW.md re-review): the just-replaced placeholder Observatory (if + # any) is now orphaned by this run -- delete it so it stops satisfying + # is_placeholder_observatory() and no longer pollutes the search-suggestion pool + # (CR-02) for the next, unrelated resolution attempt. Guarded on no other + # CampaignRun still referencing it: the same placeholder obscode can be shared by + # more than one still-unresolved row (e.g. several CSV-imported runs at one + # still-unconfigured site), so it's only safe to delete once nothing points to it + # anymore. + if previous_site_id is not None: + try: + previous_site = Observatory.objects.get(pk=previous_site_id) + except Observatory.DoesNotExist: + pass + else: + if ( + is_placeholder_observatory(previous_site) + and not CampaignRun.objects.filter(site_id=previous_site_id).exists() + ): + previous_site.delete() + + # Projection, inside its own NON-reverting try/except (never reuse the approve + # branch's revert-to-PENDING_REVIEW except block -- reverting an already-APPROVED + # run would resurrect it into the pending queue, reintroducing the dead end this + # phase closes). + try: + created = _project_calendar_event(run) + except Exception: + logger.exception('Calendar projection failed for CampaignRun %s during resolve_site.', pk) + messages.warning( + request, + "Site resolved, but the calendar entry couldn't be created automatically -- " + 'the run stays in Sites Needing Review; use Resolve to retry.', + ) + return redirect('campaigns:approval_queue') + + # Only after the projection call returned without raising: clear the flag. + run.site_needs_review = False + run.save(update_fields=['site_needs_review']) + if created: + messages.success(request, 'Site resolved — run added to the calendar.') + else: + messages.success(request, 'Site resolved.') + return redirect('campaigns:approval_queue') + + def _set_run_status(self, request, pk, action): + """D-03/D-04/D-05: mark an already-APPROVED run cancelled or weathered, and update + its linked CAMPAIGN:{pk} CalendarEvent in place if (and only if) one already exists. + + Mirrors ``_resolve_site()``'s shape: a server-side business-logic guard (never trust + the Decided-table button was only rendered for an APPROVED row -- T-23-01), then a + staleness-safe conditional queryset ``.update()`` (T-23-04/REVIEW finding #1) whose + returned row count is checked BEFORE ``run.refresh_from_db()`` or the calendar-sync + branch -- a concurrent approval_status change or row delete between the guard read + and the write must never reach ``refresh_from_db()`` (which would raise + ``CampaignRun.DoesNotExist`` on a deleted row) or silently report false success. + + A run whose window/site never projected a CAMPAIGN:{pk} event (a range/TBD run, or + one with an unresolved site) still gets its run_status set, but is never handed to + ``insert_or_create_calendar_event()`` -- that helper's create-path requires + non-nullable start_time/end_time this call deliberately omits, and would raise + (T-23-06/RESEARCH Pitfall 1). ``_project_calendar_event()`` itself is never called or + modified here. + """ + run = get_object_or_404(CampaignRun, pk=pk) + + # T-23-01: business-logic bypass guard -- only an already-APPROVED run may have its + # run_status changed via this endpoint. + if run.approval_status != CampaignRun.ApprovalStatus.APPROVED: + messages.warning(request, 'This run has not been approved yet.') + return redirect('campaigns:approval_queue') + + new_run_status = _ACTION_TO_RUN_STATUS[action] + updated_count = CampaignRun.objects.filter(pk=pk, approval_status=CampaignRun.ApprovalStatus.APPROVED).update( + run_status=new_run_status + ) + if updated_count == 0: + # REVIEW finding #1/T-23-04: the row was concurrently changed or deleted between + # the guard read above and this conditional update -- never reach + # refresh_from_db() (CampaignRun.DoesNotExist on a deleted row) or the + # calendar-sync branch below. + messages.warning(request, "This run's status could not be updated (it may have been modified or deleted).") + return redirect('campaigns:approval_queue') + + run.refresh_from_db() + + # D-05/T-23-06: only touch the linked CalendarEvent if one already exists -- never + # fabricate one for a run that never had a projected event (range/TBD/unresolved-site + # runs never reach _project_calendar_event()'s single-night+resolved-site branch). + if CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').exists(): + prefix = _RUN_STATUS_CALENDAR_PREFIX[new_run_status] + insert_or_create_calendar_event( + {'url': f'CAMPAIGN:{run.pk}'}, + fields={ + 'title': f'{prefix} {run.campaign.name}: {run.telescope_instrument}', + 'description': f'{run.observation_details}\nRun status: {run.get_run_status_display()}', + }, + ) + + messages.success(request, 'Run status updated.') + return redirect('campaigns:approval_queue') + + +def gap_analysis_available(campaign) -> bool: + """D-14: whether coverage-gap analysis makes sense for this campaign. + + False when the campaign has zero ``Target``s, or none of its ``CampaignRun``s have a + resolved ``site`` at all -- there is nothing to compute observability against either way. + Reused by Plan 03's ``CampaignRunTableView`` to gate the "Show Coverage Gaps" button + (disabled + explanatory helper text when unavailable, never a dead clickable button). + """ + if campaign.targets.count() == 0: + return False + return CampaignRun.objects.filter(campaign=campaign, site__isnull=False).exists() + + +def _as_pk_or_none(raw: str | None) -> int | None: + """Parse a raw GET-param string as a pk, or None if it isn't a valid integer (CR-01). + + Guards every `target`/`site` pk lookup in `CampaignGapAnalysisView` before it reaches + `.filter(pk=...)` -- Django's `IntegerField.get_prep_value()` raises a bare `ValueError` + for a non-integer string, which would otherwise crash the view with an unhandled 500 + instead of the documented `HttpResponseBadRequest` (T-17-01/Pitfall 3). + """ + if raw is None: + return None + try: + return int(raw) + except (TypeError, ValueError): + return None + + +class CampaignGapAnalysisView(TemplateView): + """Coverage-gap analysis page (GAP-02): observable-but-unclaimed dates for a campaign + target + site, computed on request or served from the 1-hour result cache (D-09/D-10). + + Public/read-only, same posture as ``CampaignRunTableView`` -- no ``StaffRequiredMixin``. + A plain GET (not htmx) triggers computation, per D-09; the fast per-campaign table view + never imports this module's computation path inline. Re-derives the campaign's allowed + target/site sets server-side and validates any submitted ``target``/``site`` pk against + them before either reaches a query or the cache key -- the campaign-scoped dropdown only + constrains what's *offered*, never what a raw request can submit + (``HttpResponseBadRequest`` on mismatch, T-17-01/Pitfall 3). + """ + + template_name = 'campaigns/campaignrun_gap_analysis.html' + + def get(self, request, *args, **kwargs): + """Resolve campaign/target/site/range server-side, then render the form and any result.""" + campaign = get_object_or_404(TargetList, pk=self.kwargs['pk']) + available = gap_analysis_available(campaign) + form = CampaignGapAnalysisForm(request.GET or None, campaign=campaign) + context = self.get_context_data(campaign=campaign, form=form, gap_analysis_available=available) + + if not available: + # D-14: nothing to compute -- render the disabled-state page, no computation. + return self.render_to_response(context) + + # D-12: a single-target campaign auto-uses its sole Target, ignoring any submitted + # target pk; a multi-target campaign requires one and re-validates it server-side + # against the campaign's own targets (never trusting the dropdown alone). + if campaign.targets.count() == 1: + target = campaign.targets.first() + else: + target_pk_raw = request.GET.get('target') + if not target_pk_raw: + # No selection submitted yet -- render just the form, no computation. + return self.render_to_response(context) + # CR-01: a non-numeric pk (e.g. ?target=abc) must never reach `.filter(pk=...)` + # un-guarded -- Django's IntegerField.get_prep_value() raises a bare ValueError + # for a non-integer string, which would otherwise crash this view with an + # unhandled 500 instead of the documented HttpResponseBadRequest. + target_pk = _as_pk_or_none(target_pk_raw) + target = target_pk is not None and campaign.targets.filter(pk=target_pk).first() + if not target: + # T-17-01/Pitfall 3 (IDOR): never a raw 400 page -- re-render the selection + # form with the UI-SPEC's alert-danger copy (17-03-PLAN.md Task 1). + context['idor_error'] = True + return self.render_to_response(context, status=400) + + # D-13: re-derive the campaign's allowed site set server-side (same query the form + # uses) and validate the submitted site pk is a member before using it anywhere. + site_pk_raw = request.GET.get('site') + if not site_pk_raw: + return self.render_to_response(context) + allowed_sites = Observatory.objects.filter(campaign_runs__campaign=campaign).distinct() + # CR-01/WR-04: guard the non-numeric-pk case the same way as `target` above, and + # collapse the `.exists()` + `.get()` pair into a single `.filter(...).first()` + # query to close the TOCTOU window between the existence check and the fetch. + site_pk = _as_pk_or_none(site_pk_raw) + site = site_pk is not None and allowed_sites.filter(pk=site_pk).first() + if not site: + # T-17-01/Pitfall 3 (IDOR): same treatment as the out-of-scope target case above. + context['idor_error'] = True + return self.render_to_response(context, status=400) + + # D-11/WR-03: use the already-bound, already-validated form's cleaned_data instead + # of re-parsing raw request.GET by hand -- a form validation failure now renders the + # form's own errors instead of silently substituting the 90-day default window. + if not form.is_valid(): + return self.render_to_response(context, status=400) + requested_end = form.cleaned_data.get('end_date') + start, end = clamp_date_range(date.today(), requested_end) + + result = get_or_compute_gap(campaign, target, site, start, end) + context.update({'target': target, 'site': site, 'start': start, 'end': end, 'result': result}) + return self.render_to_response(context) + + +class SiteSearchView(View): + """Shared, anonymous, throttled HTMX live-search endpoint (D-01/D-02/D-03, Phase 22 P01). + + Public/read-only, same posture as ``CampaignGapAnalysisView``/``CampaignRunTableView`` + -- deliberately no ``StaffRequiredMixin``. The candidate pool is public MPC data + (``build_site_candidates()``), and this endpoint backs the public submission form + (Plan 02) as well as the approval-queue widgets (Plan 02/03) -- neither caller is + staff-only. Returns a rendered HTML fragment (never JSON), per D-03. + """ + + http_method_names = ['get'] + + def get(self, request): + """Throttle, validate, min-length-gate, then render the suggestion fragment.""" + # D-02/Pitfall 5: staff triaging the approval queue must never trip the + # anonymous-abuse throttle meant for the public form (Assumption A3) -- exempt + # authenticated staff from the per-IP counter entirely. + client_ip = request.META.get('REMOTE_ADDR') + if not request.user.is_staff: + if client_ip: + if not _check_and_increment_throttle(client_ip): + return HttpResponse(status=429) + else: + # WR-02 (22-REVIEW.md): a missing REMOTE_ADDR must never fall back to an + # empty-string cache key -- that would silently collapse every such + # anonymous client into one shared throttle bucket (cross-client + # interference). Treat "no client IP available" as "no throttle key + # available" instead: skip throttling for this request and log it, so the + # failure mode is "no rate limit" rather than one client's usage 429-ing + # unrelated clients. + logger.warning( + 'SiteSearchView: REMOTE_ADDR missing from request.META; skipping the ' + 'per-IP throttle for this anonymous request rather than sharing a ' + 'single empty-string bucket across all such clients.' + ) + + # 22-REVIEWS.md finding 2: validate input_id server-side against a conservative + # DOM-id allowlist before it ever reaches the template context -- HTML + # auto-escaping alone is not sufficient inside the fragment's inline `onclick=` + # JS-string context (see _INPUT_ID_RE comment above). + input_id = request.GET.get('input_id', 'id_site_raw') + if not _INPUT_ID_RE.fullmatch(input_id): + input_id = 'id_site_raw' + + # 22-REVIEWS.md finding 4/T-22-02: a blank/1-char query must never reach + # build_site_candidates() -- on a cache miss that would trigger + # MPCObscodeFetcher().query_all(), and the widgets' client-side 2-char + # hx-trigger filter only gates browser-originated requests, not a direct + # anonymous GET. Gate here, AFTER the throttle check but BEFORE any pool access. + # + # gap_closure (22-04, debug/site-search-widget-query-param-mismatch.md): htmx's + # hx-get serializes only the triggering element's own name-keyed value plus + # hx-vals -- never an enclosing form's other fields, unlike POST. Neither widget + # sends `q`: the public submission form's field is `name="site_raw"` + # (campaign_forms.py) and the approval-queue/Sites-Needing-Review widgets are + # `name="site_selection"` (campaign_tables.py). Resolve the term from `q` first + # (so every existing `?q=` caller/test is unaffected), then `site_raw`, then + # `site_selection`, preferring the first non-empty value. + query = request.GET.get('q', '') or request.GET.get('site_raw', '') or request.GET.get('site_selection', '') + if len(query.strip()) < 2: + return render( + request, + 'campaigns/partials/site_search_results.html', + {'candidates': [], 'input_id': input_id, 'query': '', 'no_matches_copy': ''}, + ) + + candidates = substring_or_fuzzy_match_candidates(query, build_site_candidates()) + # Copywriting Contract: distinguish the public form (free text is fine, staff + # will resolve it) from the queue widgets (a different site is expected to + # actually resolve) by input_id. + no_matches_copy = ( + 'No matches — free text is fine, a staff member will resolve it.' + if input_id == 'id_site_raw' + else 'No matches for this search.' + ) + return render( + request, + 'campaigns/partials/site_search_results.html', + {'candidates': candidates, 'input_id': input_id, 'query': query, 'no_matches_copy': no_matches_copy}, + ) diff --git a/solsys_code/management/commands/import_campaign_csv.py b/solsys_code/management/commands/import_campaign_csv.py new file mode 100644 index 00000000..fc28b810 --- /dev/null +++ b/solsys_code/management/commands/import_campaign_csv.py @@ -0,0 +1,223 @@ +import csv +from typing import Any + +from django.core.management.base import BaseCommand, CommandError, CommandParser +from tom_targets.models import TargetList + +from solsys_code.campaign_utils import ( + insert_or_create_campaign_run, + map_observation_status, + parse_obs_window, + resolve_site, +) +from solsys_code.models import CampaignRun + +# WR-09: the D-05 natural-key columns. If the CSV's header doesn't include these exactly +# (e.g. a renamed column in a future sheet export), every row would otherwise be silently +# skipped one-by-one with no single top-level diagnostic that the header shape is wrong. +_REQUIRED_HEADERS = ('Telescope / Instrument', 'Obs. Date', 'UT Time Range') + + +class Command(BaseCommand): + """Bootstrap-import a campaign coordination CSV (e.g. the 3I/ATLAS sheet) into CampaignRun rows.""" + + help = ( + 'Bootstrap-import a campaign coordination CSV into CampaignRun rows (CAMP-04). ' + "WARNING: re-running this command over the same campaign always resets each row's " + '`target` to the auto-resolved value (D-07) -- any manual correction a staff user made ' + 'to `target` after a previous import will be silently overwritten on re-import (WR-07).' + ) + + def add_arguments(self, parser: CommandParser) -> None: + """Parse command line arguments.""" + parser.add_argument( + 'filepath', + type=str, + help='Path to the campaign coordination CSV file', + ) + parser.add_argument( + '--campaign', + type=str, + required=True, + help='Campaign TargetList name (found-or-created, D-06)', + ) + # No return statement — BaseCommand.add_arguments() returns None + + def handle(self, *args: Any, **options: Any) -> str | None: + """Import campaign CSV rows into CampaignRun, row-by-row, skip-and-log on natural-key failure. + + Only a blank Telescope / Instrument is a true natural-key failure that skips a row + (D-07); every other column defaults to a blank/None value rather than aborting the + row. `Obs. Date` never skips a row either, per D-13's never-raise contract: + `parse_obs_window()` always returns a usable window/TBD result, so every row + creates or updates a `CampaignRun` -- a resolved single-night/range window, or a + flagged TBD row (`window_needs_review=True`, counted in the summary, IMPORT-02). + Site resolution (D-08/D-09) never skips a row either -- an unresolved site is + flagged via `site_needs_review` and counted separately. + + The natural key branches on whether the row resolved to a window or TBD + (Pitfall 2, matching `CampaignRun.Meta.constraints`'s two partial + `UniqueConstraint`s exactly): a resolved window keys on `(campaign, + telescope_instrument, window_start, window_end)`; a TBD row keys on `(campaign, + telescope_instrument, contact_person)` instead, since `window_start`/`window_end` + are always `NULL` for a TBD row. A genuine same-key collision within this batch is + logged and skipped rather than silently merged into one `CampaignRun`. + + WR-07: `fields['target']` is unconditionally set to the campaign's auto-resolved + Target (D-07) on every row, every run -- including on a re-import that updates an + existing row. This is expected/acceptable for this bootstrap-import command (not + a bug), but it does mean a staff user's manual `CampaignRun.target` correction + made via the admin between imports will be reset back to the auto-resolved value + the next time this command runs over the same campaign. + + Returns: + str | None: None on completion. + """ + filepath = options['filepath'] + campaign, _ = TargetList.objects.get_or_create(name=options['campaign']) + + # D-07: single-target campaigns auto-assign that Target to every imported row. + auto_target = campaign.targets.first() if campaign.targets.count() == 1 else None + + created_count = 0 + updated_count = 0 + unchanged_count = 0 + skipped_count = 0 + site_needs_review_count = 0 + window_needs_review_count = 0 + # Two distinct key shapes (Pitfall 2): a resolved window key + # (campaign_pk, telescope_instrument, window_start, window_end), or a TBD key + # (campaign_pk, telescope_instrument, contact_person). Track keys already seen in + # this batch so a genuine duplicate is logged and skipped rather than silently + # merged into one CampaignRun via insert_or_create_campaign_run's get_or_create. + seen_window_keys: set[tuple[Any, ...]] = set() + + try: + with open(filepath, encoding='utf-8', newline='') as f: + reader = csv.DictReader(f) + # WR-09: fail fast on the header shape itself rather than silently + # skipping every row one-by-one if a required column is missing/renamed. + missing_headers = [h for h in _REQUIRED_HEADERS if h not in (reader.fieldnames or [])] + if missing_headers: + raise CommandError( + f'Campaign CSV {filepath!r} is missing required column(s): {missing_headers!r}. ' + f'Found columns: {reader.fieldnames!r}' + ) + rows = list(reader) + except OSError as exc: + raise CommandError(f'Cannot open campaign CSV {filepath!r}: {exc}') from exc + + for row_num, row in enumerate(rows, start=2): # header is row 1 + telescope_instrument = (row.get('Telescope / Instrument', '') or '').strip() + if not telescope_instrument: + # D-07: the one remaining true natural-key failure -- WR-06: log only the + # natural-key fields needed to diagnose the skip, not the full row (which + # also carries Contact Person/Email PII from the real 3I/ATLAS sheet). + self.stderr.write( + f'Row {row_num}: Telescope / Instrument is required and was blank ' + f'(Obs. Date={row.get("Obs. Date")!r})' + ) + skipped_count += 1 + continue + + # D-13: parse_obs_window() never raises -- every Obs. Date shape resolves to + # either a window (single-night or range) or the TBD tuple. + ( + window_start, + window_end, + original_obs_date_raw, + window_needs_review, + _ut_start, + _ut_end, + ut_needs_review, + ) = parse_obs_window(row.get('Obs. Date', ''), row.get('UT Time Range', '')) + if window_needs_review: + window_needs_review_count += 1 + + contact_person = row.get('Contact Person', '') or '' + + # Pitfall 2: branch the natural key on whether this row resolved to a window + # or fell through to TBD -- matches CampaignRun.Meta.constraints' two partial + # UniqueConstraints exactly (resolved: campaign+telescope_instrument+ + # window_start+window_end; TBD: campaign+telescope_instrument+contact_person). + if window_start is not None: + collision_key = (campaign.pk, telescope_instrument, window_start, window_end) + else: + collision_key = (campaign.pk, telescope_instrument, contact_person) + + if collision_key in seen_window_keys: + self.stderr.write( + f'Row {row_num}: WARNING duplicate natural key ' + f'(Telescope/Instrument={telescope_instrument!r}, ' + f'Obs. Date={row.get("Obs. Date")!r}); ' + f'skipping row to avoid merging distinct observations into one CampaignRun' + + (' (unparseable/blank UT Time Range)' if ut_needs_review else '') + ) + skipped_count += 1 + continue + seen_window_keys.add(collision_key) + + site_raw = row.get('Site Code', '') or '' + site, needs_review = resolve_site(site_raw) + if needs_review: + site_needs_review_count += 1 + + fields = { + # WR-07: unconditionally reset to auto_target on every run, including + # re-imports -- see handle()'s docstring for why this is expected. + 'target': auto_target, + 'site': site, + 'site_raw': site_raw, + 'site_needs_review': needs_review, + 'original_obs_date_raw': original_obs_date_raw, # D-04: TBD rows only, '' otherwise + 'window_needs_review': window_needs_review, + 'filters_bandpass': row.get('Filter(s)/Bandpass', '') or '', + 'observation_details': row.get('Observation Details', '') or '', + 'weather': row.get('Weather conditions or forecast', '') or '', + 'run_status': map_observation_status(row.get('Observation Status', '')), + 'approval_status': CampaignRun.ApprovalStatus.APPROVED, # D-03: bootstrap rows are vetted backfill + 'observation_outcome': row.get('Observation Outcome', '') or '', + 'publication_plans': row.get('Publication Plans', '') or '', + 'open_to_collaboration': (row.get('Open to collaboration?', '') or '').strip().lower() == 'yes', + 'contact_email': row.get('Email', '') or '', + 'comments': row.get('Other comments', '') or '', + } + + if window_start is not None: + # Resolved-window branch: contact_person is a plain field, not part of + # the lookup key. + fields['contact_person'] = contact_person + lookup = { + 'campaign': campaign, + 'telescope_instrument': telescope_instrument, + 'window_start': window_start, + 'window_end': window_end, + } + else: + # TBD branch (Pitfall 2): contact_person is promoted into the lookup key + # instead, so it's deliberately left out of `fields` to avoid + # lookup/defaults key-overlap ambiguity. + lookup = { + 'campaign': campaign, + 'telescope_instrument': telescope_instrument, + 'contact_person': contact_person, + 'window_start__isnull': True, + } + + run, action = insert_or_create_campaign_run(lookup, fields) + if action == 'created': + created_count += 1 + elif action == 'updated': + updated_count += 1 + else: + unchanged_count += 1 + + self.stdout.write( + f'Done. created: {created_count}, ' + f'updated: {updated_count}, ' + f'unchanged: {unchanged_count}, ' + f'skipped: {skipped_count}, ' + f'site_needs_review: {site_needs_review_count}, ' + f'window_needs_review: {window_needs_review_count}' + ) + return diff --git a/solsys_code/management/commands/load_telescope_runs.py b/solsys_code/management/commands/load_telescope_runs.py new file mode 100644 index 00000000..cf6c09b7 --- /dev/null +++ b/solsys_code/management/commands/load_telescope_runs.py @@ -0,0 +1,180 @@ +from datetime import date, datetime, timedelta +from datetime import timezone as dt_timezone +from typing import Any + +from django.core.management.base import BaseCommand, CommandError, CommandParser + +from solsys_code.calendar_utils import insert_or_create_calendar_event +from solsys_code.solsys_code_observatory.models import Observatory +from solsys_code.telescope_runs import ESO_NOON_TO_NOON_SITES, ParsedRun, get_site, parse_run_line, sun_event + +# The event start_time is a computed sun-event time (telescope_runs.sun_event()), not a +# stable external identifier. It drifts by a second or two between independent ingests of +# the same (site, night) because astropy's IERS Earth-orientation data (UT1-UTC / polar +# motion) is refreshed between runs (see debug/start-time-idempotency-key.md). Match an +# existing CalendarEvent whose start_time is within this window of the freshly computed +# value instead of requiring an exact datetime match, so re-ingesting an unchanged schedule +# updates the existing night rather than silently creating a near-duplicate row. The window +# is ~2 orders of magnitude larger than the largest drift observed (~2s) yet ~3 orders of +# magnitude smaller than the ~24h spacing between any two legitimately distinct events for a +# single telescope+instrument, so it can never merge two genuinely different nights. +_START_TIME_MATCH_TOLERANCE = timedelta(minutes=5) + +# Classical-schedule status -> title prefix (D-02). Only 'cancelled' has a visible +# prefix today, mirroring sync_lco_observation_calendar's _FAILURE_PREFIX_BY_STATUS +# idiom; '[CANCELLED]' is already a member of calendar_display_extras._TERMINAL_PREFIXES +# so the terminal box-shadow ring is inherited with no templatetag change. +_CLASSICAL_STATUS_PREFIX = {'cancelled': '[CANCELLED]'} + + +def _resolve_window_time(window: str, sunset, sunrise, evening_date: date) -> datetime: + """Convert a window token to a UTC datetime for a single observing night. + + Args: + window: 'BoN' for computed sunset, 'EoN' for computed sunrise, or a + 4-digit UTC HHMM string. HHMM < 1200 is treated as next-morning + UTC (evening_date + 1 day); HHMM >= 1200 is evening_date UTC. + sunset: astropy Time of sunset for this night. + sunrise: astropy Time of sunrise for this night. + evening_date: the calendar date of the observing evening. + + Returns: + datetime: UTC-aware datetime for this window boundary. + """ + upper = window.upper() + if upper == 'BON': + return sunset.to_datetime(timezone=dt_timezone.utc).replace(microsecond=0) + if upper == 'EON': + return sunrise.to_datetime(timezone=dt_timezone.utc).replace(microsecond=0) + hh, mm = int(window[:2]), int(window[2:]) + base_date = evening_date + timedelta(days=1) if hh < 12 else evening_date + return datetime(base_date.year, base_date.month, base_date.day, hh, mm, 0, tzinfo=dt_timezone.utc) + + +def _iter_run_nights(parsed: ParsedRun) -> list[date]: + """Returns one evening date per observing night, per the site's night convention. + + Las Campanas (Magellan) Start and End dates are BOTH inclusive observing + nights, so a run yields E - S + 1 nights (INGEST-01; + docs/design/telescope_runs_calendar.rst "Night convention"). ESO sites + (``ESO_NOON_TO_NOON_SITES``, e.g. NTT / La Silla) transcribe their ranges + verbatim from ESO's Tatoo tool, whose displayed END date is the noon-to-noon + closing boundary of the last night rather than an observing night itself, so + their last observing night is day2 - 1 (E - S nights). + + Args: + parsed: a ParsedRun from parse_run_line(). + + Returns: + list[date]: evening dates for each night of the run. + + Raises: + ValueError: if day2 < day1 (cross-month ranges are not supported in + Phase 3), or if an ESO noon-to-noon range leaves no observing nights + after dropping its closing boundary (day2 <= day1). + """ + if parsed.day2 < parsed.day1: + raise ValueError(f'Cross-month run ranges not yet supported in Phase 3: {parsed!r}') + n_nights = parsed.day2 - parsed.day1 + 1 + if parsed.telescope in ESO_NOON_TO_NOON_SITES: + # Tatoo's End date is the closing noon boundary of the last night, not an + # observing night -- drop it so E - S nights remain. + n_nights -= 1 + if n_nights < 1: + raise ValueError( + f'ESO noon-to-noon run range has no observing nights after dropping its ' + f'closing boundary (day1={parsed.day1}, day2={parsed.day2}): {parsed!r}' + ) + first_night = date(parsed.year, parsed.month, parsed.day1) + return [first_night + timedelta(days=i) for i in range(n_nights)] + + +class Command(BaseCommand): + """Load classical telescope run lines from a file and create or update CalendarEvents.""" + + help = 'Load classical telescope run lines from a file and create/update CalendarEvents' + + def add_arguments(self, parser: CommandParser) -> None: + """Parse command line arguments.""" + parser.add_argument( + 'filepath', + type=str, + help='Path to a text file of classical run lines (one per line)', + ) + # No return statement — BaseCommand.add_arguments() returns None + + def handle(self, *args: Any, **options: Any) -> str | None: + """Load classical schedule lines and create or update CalendarEvents. + + For each observing night derived from a run line: create a new CalendarEvent + if one does not exist, or update the existing event if any fields have changed, + or leave it untouched if nothing has changed. + + Returns: + str | None: None on completion. + """ + filepath = options['filepath'] + created_count = 0 + updated_count = 0 + unchanged_count = 0 + skipped_count = 0 + lines_processed = 0 + + try: + with open(filepath, encoding='utf-8') as f: + file_lines = list(f) + except OSError as exc: + raise CommandError(f'Cannot open schedule file {filepath!r}: {exc}') from exc + + for line_num, line in enumerate(file_lines, start=1): + if not line.strip(): + continue + lines_processed += 1 + try: + parsed = parse_run_line(line) + site = get_site(parsed.telescope) + nights = _iter_run_nights(parsed) + for d in nights: + sunset, sunrise = sun_event(site, d, 'sun') + dark_start, dark_end = sun_event(site, d, 'dark') + start_time = _resolve_window_time(parsed.start_window or 'BoN', sunset, sunrise, d) + end_time = _resolve_window_time(parsed.end_window or 'EoN', sunset, sunrise, d) + dark_start_dt = dark_start.to_datetime(timezone=dt_timezone.utc).replace(microsecond=0) + dark_end_dt = dark_end.to_datetime(timezone=dt_timezone.utc).replace(microsecond=0) + + prefix = _CLASSICAL_STATUS_PREFIX.get(parsed.status) + title = ( + f'{prefix} {parsed.telescope} {parsed.instrument}' + if prefix + else f'{parsed.telescope} {parsed.instrument}' + ) + description = ( + f'Dark window (-15 deg, UTC): {dark_start_dt.isoformat()} to {dark_end_dt.isoformat()}\n' + f'Status: {parsed.status}\n' + f'Source line: {line.strip()}' + ) + + event, action = insert_or_create_calendar_event( + {'telescope': parsed.telescope, 'instrument': parsed.instrument, 'start_time': start_time}, + {'end_time': end_time, 'title': title, 'description': description}, + start_time_tolerance=_START_TIME_MATCH_TOLERANCE, + ) + if action == 'created': + created_count += 1 + elif action == 'updated': + updated_count += 1 + else: + unchanged_count += 1 + except (ValueError, Observatory.DoesNotExist) as exc: + self.stderr.write(f'Line {line_num}: {exc} (line text: {line.strip()!r})') + skipped_count += 1 + continue + + self.stdout.write( + f'Done. lines processed: {lines_processed}, ' + f'created: {created_count}, ' + f'updated: {updated_count}, ' + f'unchanged: {unchanged_count}, ' + f'skipped: {skipped_count}' + ) + return diff --git a/solsys_code/management/commands/sync_gemini_observation_calendar.py b/solsys_code/management/commands/sync_gemini_observation_calendar.py new file mode 100644 index 00000000..30360bb4 --- /dev/null +++ b/solsys_code/management/commands/sync_gemini_observation_calendar.py @@ -0,0 +1,192 @@ +"""Management command to sync Gemini queue ObservationRecords to CalendarEvents.""" + +import logging +from datetime import datetime, timedelta +from datetime import timezone as dt_timezone +from typing import Any + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandParser +from tom_observations.models import ObservationRecord + +from solsys_code.calendar_utils import insert_or_create_calendar_event + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + """Sync Gemini queue ObservationRecords to the FOMO calendar as CalendarEvents.""" + + help = 'Sync Gemini queue ObservationRecords to CalendarEvents' + + def add_arguments(self, parser: CommandParser) -> None: + """Parse command line arguments.""" + pass + + def handle(self, *args: Any, **options: Any) -> str | None: + """Sync all GEM ObservationRecords to CalendarEvents. + + For each ObservationRecord with facility='GEM', derives CalendarEvent fields + (telescope, instrument, proposal, title, window) from the record's parameters + JSON and settings.FACILITIES['GEM']['programs'], then creates or updates the + event idempotently using a no-churn get_or_create + update_fields idiom. + + Password key is stripped from parameters immediately at record load time (D-04) + and never reaches stdout, stderr, or any CalendarEvent field (GEM-SECURE-01). + + Returns: + None + """ + records = ObservationRecord.objects.filter(facility='GEM') + counters: dict[str, dict[str, int]] = { + 'GS': {'created': 0, 'updated': 0, 'unchanged': 0, 'skipped': 0}, + 'GN': {'created': 0, 'updated': 0, 'unchanged': 0, 'skipped': 0}, + } + + for record in records: + # D-04: strip password immediately, before any logging or field derivation. + safe_params = {k: v for k, v in (record.parameters or {}).items() if k != 'password'} + + # GEM-TELE-01: derive site and telescope name from program prefix. + prog = safe_params.get('prog', '') + if prog.startswith('GS-'): + site_key = 'GS' + telescope = 'Gemini South' + elif prog.startswith('GN-'): + site_key = 'GN' + telescope = 'Gemini North' + else: + self.stderr.write(f'Unknown Gemini program prefix in {prog!r}; skipping ObservationRecord {record.pk}') + counters.setdefault('UNKNOWN', {'created': 0, 'updated': 0, 'unchanged': 0, 'skipped': 0}) + counters['UNKNOWN']['skipped'] += 1 + continue + + try: + # D-03: use first obsid entry; warn when multiple are present. + obsid_list = safe_params['obsid'] + if not isinstance(obsid_list, list): + self.stderr.write( + f'ObservationRecord pk={record.pk}: obsid must be a list, ' + f'got {type(obsid_list).__name__!r} — skipping' + ) + counters[site_key]['skipped'] += 1 + continue + if not obsid_list: + logger.warning( + 'ObservationRecord pk=%s has empty obsid list — skipping', + record.pk, + ) + counters[site_key]['skipped'] += 1 + continue + if len(obsid_list) > 1: + logger.warning( + 'ObservationRecord pk=%s has multiple obsid entries: %r — using first entry only', + record.pk, + obsid_list, + ) + obs_code = obsid_list[0] + + # GEM-INSTR-01 / D-02: look up instrument description and ToO-type prefix from settings. + gem_programs = settings.FACILITIES.get('GEM', {}).get('programs', {}) + description_str: str | None = gem_programs.get(prog, {}).get(obs_code) + + # Determine whether an explicit window is present (GEM-WINDOW-01). + window_date = safe_params.get('windowDate') + window_time_str = safe_params.get('windowTime') + window_duration = safe_params.get('windowDuration') + has_explicit_window = bool(window_date and window_time_str and window_duration) + + if description_str is not None: + # Strip the 'Std: ' or 'Rap: ' prefix to get the instrument label. + instrument = description_str.split(': ', 1)[1] if ': ' in description_str else description_str + elif has_explicit_window: + # GEM-INSTR-01 raw fallback: explicit window present but obs code absent from settings. + # Use the raw obs code as the instrument label rather than skipping the record. + instrument = obs_code + else: + # D-01: no explicit window and obs code absent from settings — ToO-type is unknowable. + # An event with unknown time bounds would be misleading; skip and count. + logger.warning( + "%r obs code %r not found in FACILITIES['GEM']['programs'] — skipping ObservationRecord %s", + prog, + obs_code, + record.pk, + ) + counters[site_key]['skipped'] += 1 + continue + + # Derive the observing window. + if has_explicit_window: + # GEM-WINDOW-01: parse explicit date + time + duration from parameters. + start_dt = datetime.strptime(window_date, '%Y-%m-%d').replace(tzinfo=dt_timezone.utc) + time_dt = datetime.strptime(window_time_str, '%H:%M') + start_time = start_dt.replace(hour=time_dt.hour, minute=time_dt.minute) + end_time = start_time + timedelta(hours=float(window_duration)) + else: + # GEM-WINDOW-02: fall back to ToO-type prefix when no explicit window is present. + # description_str is guaranteed non-None here (raw fallback took the explicit-window branch). + if description_str.startswith('Rap:'): + start_time = record.created + end_time = record.created + timedelta(hours=24) + elif description_str.startswith('Std:'): + start_time = record.created + timedelta(hours=24) + end_time = record.created + timedelta(days=7) + else: + logger.warning( + 'Unrecognised ToO-type prefix in %r for ObservationRecord %s — skipping', + description_str, + record.pk, + ) + counters[site_key]['skipped'] += 1 + continue + + # GEM-STATUS-01: prefix title with [ON_HOLD] when ready == 'false'. + # Use str().lower() to handle both boolean False and string 'false' from the JSONField. + ready = safe_params.get('ready', 'true') + title_prefix = '[ON_HOLD] ' if str(ready).lower() == 'false' else '' + title = f'{title_prefix}{telescope} {instrument} ToO' + + # GEM-KEY-01: stable, human-readable URL key. + url = f'GEM:{prog}/{record.observation_id}' + + fields: dict[str, Any] = { + 'start_time': start_time, + 'end_time': end_time, + 'title': title, + 'telescope': telescope, + 'instrument': instrument, + 'proposal': prog, + } + + # GEM-NOCHURN-01: delegate create-or-update to the shared helper; + # only saves when something actually changed. + _event, action = insert_or_create_calendar_event({'url': url}, fields) + counters[site_key][action] += 1 + + except (KeyError, ValueError) as exc: + # Never interpolate safe_params or record.parameters into this message (GEM-SECURE-01). + # Emit only the exception class name on stderr; strptime errors embed the offending + # input value in their message, which could expose parameter content via {exc}. + self.stderr.write(f'Skipping observation_id={record.observation_id!r}: {type(exc).__name__}') + logger.debug('Full exception for ObservationRecord %s: %s', record.pk, exc) + counters[site_key]['skipped'] += 1 + continue + + # D-08: two-line per-site summary mirroring the LCO sync format. + self.stdout.write( + f'Gemini South: created: {counters["GS"]["created"]}, ' + f'updated: {counters["GS"]["updated"]}, ' + f'unchanged: {counters["GS"]["unchanged"]}, ' + f'skipped: {counters["GS"]["skipped"]}' + ) + self.stdout.write( + f'Gemini North: created: {counters["GN"]["created"]}, ' + f'updated: {counters["GN"]["updated"]}, ' + f'unchanged: {counters["GN"]["unchanged"]}, ' + f'skipped: {counters["GN"]["skipped"]}' + ) + unknown_skipped = counters.get('UNKNOWN', {}).get('skipped', 0) + if unknown_skipped: + self.stdout.write(f'Unknown prefix: skipped: {unknown_skipped}') + self.stdout.write('Done.') + return None diff --git a/solsys_code/management/commands/sync_lco_observation_calendar.py b/solsys_code/management/commands/sync_lco_observation_calendar.py new file mode 100644 index 00000000..693cb4bc --- /dev/null +++ b/solsys_code/management/commands/sync_lco_observation_calendar.py @@ -0,0 +1,372 @@ +from datetime import datetime +from datetime import timezone as dt_timezone +from typing import Any + +from django.core.management.base import BaseCommand, CommandParser +from tom_observations.facilities.lco import LCOFacility +from tom_observations.facilities.soar import SOARFacility +from tom_observations.models import ObservationRecord + +from solsys_code.calendar_utils import ( + InstrumentExtractionError, + _coarse_telescope_label, + _derive_telescope, + _extract_instrument, + _resolve_placement_block, + insert_or_create_calendar_event, +) +from solsys_code.models import CalendarEventTelescopeLabel + +# TERM-01/D-04: terminal-failure status -> title prefix. COMPLETED is deliberately +# absent here (D-06 research correction) — it is terminal per +# LCOFacility().get_terminal_observing_states() (5 states) but is NOT one of the 4 +# failure states returned by LCOFacility().get_failed_observing_states(), so it gets +# a clean title, same as a normally-placed record. This is a hand-typed snapshot of +# the library's current 4 failure states, not auto-derived — if LCOFacility ever adds +# a new failure state, update this dict too (the fallback below still tags it +# '[FAILED]' so a sync never silently skips a real failure state). +_FAILURE_PREFIX_BY_STATUS = { + 'WINDOW_EXPIRED': '[EXPIRED]', + 'CANCELED': '[CANCELLED]', + 'FAILURE_LIMIT_REACHED': '[FAILED]', + 'NOT_ATTEMPTED': '[FAILED]', +} + + +def _failure_prefix(status: str, facility: LCOFacility) -> str | None: + """Return the terminal-failure title prefix for a status, or None if not a failure state. + + Args: + status: the ObservationRecord's status string. + facility: a shared LCOFacility instance. + + Returns: + str | None: the TERM-01 prefix (e.g. '[EXPIRED]') if status is one of + facility.get_failed_observing_states(), else None. + """ + if status not in set(facility.get_failed_observing_states()): + return None + return _FAILURE_PREFIX_BY_STATUS.get(status, '[FAILED]') + + +def _title_for( + record: ObservationRecord, telescope: str, instrument: str, facility: LCOFacility, label_was_fallback: bool +) -> str: + """Build the CalendarEvent title for a record (D-03/D-04/D-06/D-09). + + Args: + record: the ObservationRecord being synced. + telescope: derived telescope label. + instrument: instrument_type from the record's parameters. + facility: a shared LCOFacility instance. + label_was_fallback: True if telescope is a coarse fallback label for a + PLACED record whose live API resolution failed/timed out/returned an + unmapped code (D-07) -- never True for a banner-stage record. + + Returns: + str: the title, with a terminal-failure prefix, '[QUEUED]' prefix, + '[UNVERIFIED]' prefix, or clean (no prefix), in that priority order + (D-09): a terminal-failure prefix always wins, even over + '[UNVERIFIED]'; '[QUEUED]' (banner stage) and '[UNVERIFIED]' (placed + + fallback) are mutually exclusive by construction since '[UNVERIFIED]' + only ever applies to a placed record (D-07); clean (no prefix) is a + placed record whose label was resolved via the live API successfully. + """ + prefix = _failure_prefix(record.status, facility) + if prefix is not None: + return f'{prefix} {telescope} {instrument}' + if record.scheduled_start is None: + return f'[QUEUED] {telescope} {instrument}' + if label_was_fallback: + return f'[UNVERIFIED] {telescope} {instrument}' + return f'{telescope} {instrument}' + + +def _time_window(record: ObservationRecord) -> tuple[datetime, datetime]: + """Derive the active start/end time window for a record (SYNC-02/SYNC-03). + + Args: + record: the ObservationRecord being synced. + + Returns: + tuple[datetime, datetime]: (start_time, end_time), timezone-aware UTC. + + Raises: + KeyError: if scheduled_start is None and parameters lacks 'start'/'end'. + ValueError: if parameters['start']/['end'] are not valid ISO datetime strings, + or if scheduled_start/scheduled_end are inconsistently populated (one set, + the other None) — a state CalendarEvent's non-nullable times cannot accept. + """ + if record.scheduled_start is None and record.scheduled_end is None: + # parameters['start']/['end'] are naive ISO strings (Pitfall 3) -- attach UTC + # explicitly since LCO request-submission times are conventionally UTC. + start_time = datetime.fromisoformat(record.parameters['start']).replace(tzinfo=dt_timezone.utc) + end_time = datetime.fromisoformat(record.parameters['end']).replace(tzinfo=dt_timezone.utc) + elif record.scheduled_start is not None and record.scheduled_end is not None: + start_time = record.scheduled_start + end_time = record.scheduled_end + else: + raise ValueError( + f'Inconsistent schedule state: scheduled_start={record.scheduled_start!r}, ' + f'scheduled_end={record.scheduled_end!r}' + ) + return start_time, end_time + + +def _build_event_fields(record: ObservationRecord, facility: LCOFacility) -> dict[str, Any]: + """Build the full set of CalendarEvent field values for a record. + + Implements the TELESCOPE-02/03/04 decision tree (D-01/D-02/D-07, Pitfall 4): a + banner-stage record (scheduled_start is None) gets the coarse fallback label + with no API call (D-01) and is never counted/flagged as a failure (D-02/D-07). A + placed record attempts a single live API resolution via + _resolve_placement_block; an API failure/timeout AND a successfully-returned but + unmapped (site, telescope_code) pair are the SAME fallback bucket (Pitfall 4) -- + both set label_was_fallback=True, route to the coarse label, and increment the + same telescope_api_failed counter. + + Args: + record: the ObservationRecord being synced. + facility: a shared LCOFacility instance. + + Returns: + dict[str, Any]: keyword args for CalendarEvent (url, title, description, + start_time, end_time, telescope, instrument, proposal), plus a + 'telescope_api_failed' bool key that the caller (Command.handle()) pops + before constructing CalendarEvent kwargs, exactly like 'url' is already + popped. + + Raises: + KeyError: if a required parameters key (proposal/start/end) is missing. + ValueError: if parameters['start']/['end'] cannot be parsed as datetimes. + InstrumentExtractionError: if _extract_instrument (D-01..D-06) finds no + science config and no exposure-signal config anywhere in parameters. + """ + instrument = _extract_instrument(record.parameters) + if instrument is None: + raise InstrumentExtractionError( + f'No recognized configuration_type or exposure signal found in observation_id=' + f'{record.observation_id!r} parameters' + ) + coarse = _coarse_telescope_label(instrument, record.facility) + + if record.scheduled_start is None: + # D-01: banner stage -- no API call attempted; D-02/D-07: never counted as a + # failure and never gets the [UNVERIFIED] prefix. + telescope = coarse + label_was_fallback = False + else: + block = _resolve_placement_block(record.observation_id, facility) + # T-07-03: a malformed/tampered API block validates 'state' upstream but never + # 'site'/'telescope' -- read via .get() so a missing key yields None and routes + # to the same coarse-fallback bucket as an unmapped pair, instead of raising + # KeyError into the generic except clause one layer up in handle(). + resolved = _derive_telescope(block.get('site'), block.get('telescope')) if block is not None else None + if resolved is None: + # Pitfall 4: an API call failure/timeout (block is None) and a + # successfully-returned but unmapped (site, telescope_code) pair + # (resolved is None) are the SAME fallback bucket. + telescope = coarse + label_was_fallback = True + else: + telescope = resolved + label_was_fallback = False + + proposal = record.parameters['proposal'] + url = facility.get_observation_url(record.observation_id) + start_time, end_time = _time_window(record) + title = _title_for(record, telescope, instrument, facility, label_was_fallback) + description = ( + f'Proposal: {proposal}\n' + f'Status: {record.status}\n' + f'Window (UTC): {start_time.strftime("%Y-%m-%dT%H:%M:%S")} to {end_time.strftime("%Y-%m-%dT%H:%M:%S")}' + ) + if label_was_fallback: + # TELESCOPE-04/SYNC-09: a generic, never-exception-derived note. Not logged + # here -- Command.handle() owns the stderr log line (caller-logging + # discipline kept in one place). + description += '\nTelescope label unverified: live API lookup failed or returned an unmapped code.' + return { + 'url': url, + 'title': title, + 'description': description, + 'start_time': start_time, + 'end_time': end_time, + 'telescope': telescope, + 'instrument': instrument, + 'proposal': proposal, + # D-02 scope: True only for a PLACED record whose label was a fallback -- + # never True for a banner-stage record. Popped by handle() before + # constructing CalendarEvent kwargs, mirroring 'url'. + 'telescope_api_failed': record.scheduled_start is not None and label_was_fallback, + } + + +def _parse_proposal_arg(raw: str) -> list[str] | None: + """Parse the --proposal argument into a deduped code list, or the ALL sentinel. + + Args: + raw: the raw --proposal argument value (e.g. 'A,B,C', 'ALL', 'A,A,B,'). + + Returns: + list[str] | None: None if raw is the case-insensitive 'all' token + (SELECT-03/D-02 -- sync every record regardless of proposal). Otherwise + a list of proposal codes, comma-split, stripped, with empty segments + dropped and duplicates removed while preserving first-seen order + (D-03). Codes keep their original casing -- proposal codes are + case-SENSITIVE (D-01), so this never .upper()/.lower()s a code. + """ + if raw.strip().lower() == 'all': + return None + seen: dict[str, None] = {} + for segment in raw.split(','): + code = segment.strip() + if not code: + continue + seen.setdefault(code, None) + return list(seen) + + +class Command(BaseCommand): + """Sync LCO queue ObservationRecords to the FOMO calendar as CalendarEvents.""" + + help = 'Sync LCO queue ObservationRecords for a proposal to CalendarEvents' + + def add_arguments(self, parser: CommandParser) -> None: + """Parse command line arguments.""" + parser.add_argument( + '--proposal', + type=str, + required=True, + help=( + 'LCO/SOAR proposal code(s) to filter ObservationRecords by. Accepts a ' + "single code, a comma-separated list (e.g. 'A,B,C'), or the case-" + "insensitive token 'ALL' to sync every record regardless of proposal." + ), + ) + + def handle(self, *args: Any, **options: Any) -> str | None: + """Sync matching LCO/SOAR ObservationRecords to CalendarEvents. + + For each ObservationRecord(facility__in=['LCO', 'SOAR']) matching the + --proposal selection (a comma-separated code list, or every record when + --proposal is the ALL sentinel): create a new CalendarEvent if one does not + exist (keyed on url), or update the existing event in place if any fields + changed, or leave it untouched if nothing changed (SYNC-04 no-churn + idempotency). Each record is dispatched through the facility instance + matching its own `facility` value (SELECT-05) -- never a single shared + instance reused across both LCO and SOAR records. + + Returns: + str | None: None on completion. + """ + proposal = options['proposal'] + # Eager dispatch dict, both keys unconditionally (D-06): each record is + # processed via the facility instance matching its own `facility` value, + # never a single shared instance reused across LCO and SOAR (SELECT-05). + facilities = {'LCO': LCOFacility(), 'SOAR': SOARFacility()} + + # Per-facility counters (D-08): every facility's created/updated/unchanged/ + # skipped/extraction_failed/telescope_api_failed counts must be individually + # visible in the summary line. 'extraction_failed' (D-06) and + # 'telescope_api_failed' (SYNC-06/D-02) are dedicated counters distinct from + # 'skipped' and from each other. + counters = { + 'LCO': { + 'created': 0, + 'updated': 0, + 'unchanged': 0, + 'skipped': 0, + 'extraction_failed': 0, + 'telescope_api_failed': 0, + }, + 'SOAR': { + 'created': 0, + 'updated': 0, + 'unchanged': 0, + 'skipped': 0, + 'extraction_failed': 0, + 'telescope_api_failed': 0, + }, + } + + records = ObservationRecord.objects.filter(facility__in=['LCO', 'SOAR']) + codes = _parse_proposal_arg(proposal) + if codes is not None: + records = records.filter(parameters__proposal__in=codes) + + for record in records: + facility = facilities.get(record.facility) + if facility is None: + # D-07 defensive path: an unexpected facility value on a row that + # otherwise matched facility__in=['LCO', 'SOAR'] shouldn't happen, + # but skip-and-log rather than abort the whole run. + self.stderr.write( + f'Skipping observation_id={record.observation_id!r}: unrecognized facility {record.facility!r}' + ) + counters.setdefault( + record.facility, + { + 'created': 0, + 'updated': 0, + 'unchanged': 0, + 'skipped': 0, + 'extraction_failed': 0, + 'telescope_api_failed': 0, + }, + ) + counters[record.facility]['skipped'] += 1 + continue + + try: + fields = _build_event_fields(record, facility) + except InstrumentExtractionError as exc: + # D-06: a fully-malformed record (no recognized configuration_type, no + # exposure signal anywhere) is counted separately from 'skipped'. + self.stderr.write(f'Skipping observation_id={record.observation_id!r}: {exc}') + counters[record.facility]['extraction_failed'] += 1 + continue + except (KeyError, ValueError) as exc: + self.stderr.write(f'Skipping observation_id={record.observation_id!r}: {exc}') + counters[record.facility]['skipped'] += 1 + continue + + url = fields.pop('url') + telescope_api_failed = fields.pop('telescope_api_failed') + if telescope_api_failed: + # SYNC-09/D-11: fixed, generic message -- never interpolates a + # caught exception (no {exc}/str(exc)/repr(exc) here). SYNC-07: the + # record still gets a CalendarEvent below; the run continues. + self.stderr.write( + f'Telescope API lookup failed or returned an unmapped code for ' + f'observation_id={record.observation_id!r}; using fallback label.' + ) + counters[record.facility]['telescope_api_failed'] += 1 + + event, action = insert_or_create_calendar_event({'url': url}, fields) + counters[record.facility][action] += 1 + + # Phase 8 / DISPLAY-01: always reconcile the sidecar row to the current + # telescope_api_failed signal, regardless of whether CalendarEvent's own + # fields changed -- kept as a separate statement, never folded into + # `fields` or `changed`. is_verified reflects the outcome of the most + # recent sync run that included this record, not real-time state. + CalendarEventTelescopeLabel.objects.update_or_create( + event=event, defaults={'is_verified': not telescope_api_failed} + ) + + # D-08: per-facility breakdown. Each facility's six counts use the same + # 'created: N' / 'updated: N' / 'unchanged: N' / 'skipped: N' / + # 'extraction_failed: N' / 'telescope_api_failed: N' phrasing as the prior + # single-facility summary line, kept per-facility for visibility. + # extraction_failed (D-06) and telescope_api_failed (SYNC-06) are each + # distinct from skipped and from each other. + summary = ' | '.join( + f'{facility_name}: created: {counts["created"]}, updated: {counts["updated"]}, ' + f'unchanged: {counts["unchanged"]}, skipped: {counts["skipped"]}, ' + f'extraction_failed: {counts["extraction_failed"]}, ' + f'telescope_api_failed: {counts["telescope_api_failed"]}' + for facility_name, counts in counters.items() + ) + self.stdout.write(f'Done. proposal: {proposal}, {summary}') + return diff --git a/solsys_code/migrations/0001_calendareventtelescopelabel.py b/solsys_code/migrations/0001_calendareventtelescopelabel.py new file mode 100644 index 00000000..d5c465fb --- /dev/null +++ b/solsys_code/migrations/0001_calendareventtelescopelabel.py @@ -0,0 +1,23 @@ +# Generated by Django 5.2.14 on 2026-06-25 05:32 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('tom_calendar', '0005_calendarevent_instrument'), + ] + + operations = [ + migrations.CreateModel( + name='CalendarEventTelescopeLabel', + fields=[ + ('event', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='telescope_label_meta', serialize=False, to='tom_calendar.calendarevent', verbose_name='Calendar event')), + ('is_verified', models.BooleanField(default=True, verbose_name='Whether the telescope label was live-verified against the LCO API')), + ], + ), + ] diff --git a/solsys_code/migrations/0002_campaignrun.py b/solsys_code/migrations/0002_campaignrun.py new file mode 100644 index 00000000..7f5b8faf --- /dev/null +++ b/solsys_code/migrations/0002_campaignrun.py @@ -0,0 +1,116 @@ +# Generated by Django 5.2.15 on 2026-07-03 06:07 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('solsys_code', '0001_calendareventtelescopelabel'), + ('solsys_code_observatory', '0002_observatory_timezone_seed'), + ('tom_targets', '0030_alter_basetarget_slope'), + ] + + operations = [ + migrations.CreateModel( + name='CampaignRun', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('telescope_instrument', models.CharField(max_length=255, verbose_name='Telescope / instrument')), + ( + 'site_raw', + models.CharField(blank=True, default='', max_length=255, verbose_name='Original site code text'), + ), + ( + 'site_needs_review', + models.BooleanField( + default=False, + verbose_name='Whether the site could not be automatically resolved and needs manual review', + ), + ), + ('obs_date', models.DateField(blank=True, null=True, verbose_name='Observation date')), + ('ut_start', models.DateTimeField(blank=True, null=True, verbose_name='UT start time')), + ('ut_end', models.DateTimeField(blank=True, null=True, verbose_name='UT end time')), + ( + 'filters_bandpass', + models.CharField(blank=True, default='', max_length=255, verbose_name='Filter(s) / bandpass'), + ), + ('observation_details', models.TextField(blank=True, default='', verbose_name='Observation details')), + ('weather', models.TextField(blank=True, default='', verbose_name='Weather conditions or forecast')), + ('observation_outcome', models.TextField(blank=True, default='', verbose_name='Observation outcome')), + ('publication_plans', models.TextField(blank=True, default='', verbose_name='Publication plans')), + ('open_to_collaboration', models.BooleanField(default=False, verbose_name='Open to collaboration?')), + ('comments', models.TextField(blank=True, default='', verbose_name='Other comments')), + ( + 'contact_person', + models.CharField(blank=True, default='', max_length=255, verbose_name='Contact person'), + ), + ( + 'contact_email', + models.EmailField(blank=True, default='', max_length=254, verbose_name='Contact email'), + ), + ( + 'approval_status', + models.CharField( + choices=[ + ('pending_review', 'Pending Review'), + ('approved', 'Approved'), + ('rejected', 'Rejected'), + ], + default='pending_review', + max_length=20, + verbose_name='Approval status', + ), + ), + ( + 'run_status', + models.CharField( + choices=[ + ('requested', 'Requested'), + ('planned', 'Planned'), + ('observed', 'Observed'), + ('reduced', 'Reduced'), + ('published', 'Published'), + ('cancelled', 'Cancelled'), + ('not_awarded', 'Not Awarded'), + ('weather_tech_failure', 'Weather/Technical Failure'), + ], + default='requested', + max_length=30, + verbose_name='Run status', + ), + ), + ( + 'campaign', + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name='campaign_runs', + to='tom_targets.targetlist', + verbose_name='Campaign target list', + ), + ), + ( + 'site', + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='campaign_runs', + to='solsys_code_observatory.observatory', + verbose_name='Resolved observing site', + ), + ), + ( + 'target', + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='campaign_runs', + to='tom_targets.basetarget', + verbose_name='Observed target', + ), + ), + ], + ), + ] diff --git a/solsys_code/migrations/0003_campaignrun_natural_key_unique_constraint.py b/solsys_code/migrations/0003_campaignrun_natural_key_unique_constraint.py new file mode 100644 index 00000000..d06c046c --- /dev/null +++ b/solsys_code/migrations/0003_campaignrun_natural_key_unique_constraint.py @@ -0,0 +1,20 @@ +# Generated by Django 5.2.15 on 2026-07-03 08:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('solsys_code', '0002_campaignrun'), + ('solsys_code_observatory', '0002_observatory_timezone_seed'), + ('tom_targets', '0030_alter_basetarget_slope'), + ] + + operations = [ + migrations.AddConstraint( + model_name='campaignrun', + constraint=models.UniqueConstraint( + fields=('campaign', 'telescope_instrument', 'ut_start'), name='unique_campaign_run_natural_key' + ), + ), + ] diff --git a/solsys_code/migrations/0004_campaignrun_window_schema.py b/solsys_code/migrations/0004_campaignrun_window_schema.py new file mode 100644 index 00000000..c6d9c2d7 --- /dev/null +++ b/solsys_code/migrations/0004_campaignrun_window_schema.py @@ -0,0 +1,144 @@ +# Generated by Django 5.2.15 on 2026-07-09, hand-edited per +# .planning/phases/19-window-schema-migration/19-RESEARCH.md Pattern 1 to insert the three +# RunPython data-migration steps (backfill + TBD-branch dedup + resolved-window-branch dedup) +# in the load-bearing position: after the new fields are added, before the old +# constraint/fields are removed, and before the two new partial constraints are added +# (Pitfall 3 -- each dedup step must precede its matching AddConstraint or that step raises +# IntegrityError against real leftover duplicate rows -- CR-01: the resolved-window branch +# needs its own dedup for the same reason the TBD branch does, since backfill_window_fields +# collapses the old ut_start-distinguished rows onto the same (window_start, window_end) +# pair). D-02: single combined, non-reversible migration. + +import logging + +from django.db import migrations, models +from django.db.models import F + +logger = logging.getLogger(__name__) + + +def backfill_window_fields(apps, schema_editor): + """SCHED-05: window_start=window_end=obs_date for every row (NULL stays NULL -> TBD).""" + CampaignRun = apps.get_model('solsys_code', 'CampaignRun') + CampaignRun.objects.all().update(window_start=F('obs_date'), window_end=F('obs_date')) + + +def dedupe_tbd_collisions(apps, schema_editor): + """D-07/D-08: generic cleanup of (campaign, telescope_instrument, contact_person) + collisions among window_start IS NULL rows -- keeps the lowest pk, deletes the rest, + logs what was removed. Must run before the new TBD-branch UniqueConstraint is added + below, or that AddConstraint would fail against real leftover-fixture duplicates. + """ + CampaignRun = apps.get_model('solsys_code', 'CampaignRun') + seen: dict[tuple, int] = {} + # order_by('pk') is what makes "keep the lowest pk" correct: the first row seen for + # a given key is always the lowest-pk one, so every later match is a dup to delete. + for run in CampaignRun.objects.filter(window_start__isnull=True).order_by('pk'): + key = (run.campaign_id, run.telescope_instrument, run.contact_person) + if key in seen: + logger.warning( + 'Deleting duplicate TBD CampaignRun pk=%s (kept pk=%s) for ' + 'campaign=%s telescope_instrument=%r contact_person=%r', + run.pk, + seen[key], + run.campaign_id, + run.telescope_instrument, + run.contact_person, + ) + run.delete() + else: + seen[key] = run.pk + + +def dedupe_resolved_window_collisions(apps, schema_editor): + """CR-01: analogous to dedupe_tbd_collisions, but for the resolved-window branch. + + Two pre-migration rows that shared campaign+telescope_instrument+obs_date but differed + only by the now-dropped ut_start collapse onto an identical (window_start, window_end) + tuple after backfill_window_fields runs. Must run before the resolved-window + UniqueConstraint is added below, or that AddConstraint can fail against real + pre-existing same-night, different-ut_start rows. + """ + CampaignRun = apps.get_model('solsys_code', 'CampaignRun') + seen: dict[tuple, int] = {} + for run in CampaignRun.objects.filter(window_start__isnull=False).order_by('pk'): + key = (run.campaign_id, run.telescope_instrument, run.window_start, run.window_end) + if key in seen: + logger.warning( + 'Deleting duplicate resolved-window CampaignRun pk=%s (kept pk=%s) for ' + 'campaign=%s telescope_instrument=%r window=%s..%s', + run.pk, + seen[key], + run.campaign_id, + run.telescope_instrument, + run.window_start, + run.window_end, + ) + run.delete() + else: + seen[key] = run.pk + + +class Migration(migrations.Migration): + dependencies = [ + ('solsys_code', '0003_campaignrun_natural_key_unique_constraint'), + ('solsys_code_observatory', '0002_observatory_timezone_seed'), + ('tom_targets', '0030_alter_basetarget_slope'), + ] + + operations = [ + # 1-2: new nullable fields, added first so RunPython below can populate them. + migrations.AddField( + model_name='campaignrun', + name='window_start', + field=models.DateField(blank=True, null=True, verbose_name='Observing window start'), + ), + migrations.AddField( + model_name='campaignrun', + name='window_end', + field=models.DateField(blank=True, null=True, verbose_name='Observing window end'), + ), + # 3-5: data migration, in this order (backfill before either dedup -- both dedup + # steps key on window_start, which is only meaningful after backfill runs; CR-01: + # the resolved-window dedup must also run before the resolved-window AddConstraint + # below, same as the TBD dedup must run before the TBD AddConstraint). + migrations.RunPython(backfill_window_fields, reverse_code=migrations.RunPython.noop), + migrations.RunPython(dedupe_tbd_collisions, reverse_code=migrations.RunPython.noop), + migrations.RunPython(dedupe_resolved_window_collisions, reverse_code=migrations.RunPython.noop), + # 6: old constraint removed BEFORE the fields it references are removed. + migrations.RemoveConstraint( + model_name='campaignrun', + name='unique_campaign_run_natural_key', + ), + # 7: old fields dropped (D-01 hard cutover). + migrations.RemoveField( + model_name='campaignrun', + name='obs_date', + ), + migrations.RemoveField( + model_name='campaignrun', + name='ut_end', + ), + migrations.RemoveField( + model_name='campaignrun', + name='ut_start', + ), + # 8-9: new partial constraints -- resolved-window branch keyed on all four fields; + # TBD branch keyed on (campaign, telescope_instrument, contact_person) only. + migrations.AddConstraint( + model_name='campaignrun', + constraint=models.UniqueConstraint( + condition=models.Q(('window_start__isnull', False)), + fields=('campaign', 'telescope_instrument', 'window_start', 'window_end'), + name='unique_campaign_run_resolved_window', + ), + ), + migrations.AddConstraint( + model_name='campaignrun', + constraint=models.UniqueConstraint( + condition=models.Q(('window_start__isnull', True)), + fields=('campaign', 'telescope_instrument', 'contact_person'), + name='unique_campaign_run_tbd_natural_key', + ), + ), + ] diff --git a/solsys_code/migrations/0005_campaignrun_campaign_run_window_start_end_null_together.py b/solsys_code/migrations/0005_campaignrun_campaign_run_window_start_end_null_together.py new file mode 100644 index 00000000..5f57279a --- /dev/null +++ b/solsys_code/migrations/0005_campaignrun_campaign_run_window_start_end_null_together.py @@ -0,0 +1,68 @@ +# Generated by Django 5.2.15 on 2026-07-10 07:24, hand-edited per +# .planning/phases/19-window-schema-migration/19-REVIEW.md WR-02 to add a defensive +# RunPython cleanup step ahead of the AddConstraint below, mirroring 0004's own CR-01 +# precedent for the same class of risk (adding a constraint against data that may not +# actually satisfy it yet raises an unhandled IntegrityError with no recovery path). + +import logging + +from django.db import migrations, models +from django.db.models import Q + +logger = logging.getLogger(__name__) + + +def normalize_mismatched_window_pairs(apps, schema_editor): + """WR-02: normalize any single-sided window_start/window_end pair to fully-TBD + (both NULL) before the campaign_run_window_start_end_null_together CheckConstraint + is added below. + + In the current codebase this is a no-op: 0004's backfill_window_fields sets both + fields from the same source column (F('obs_date')), and no write path + (CampaignRunSubmissionView, import_campaign_csv) ever sets them independently, so + the invariant already holds by the time this migration runs in the same deploy as + 0004. This step exists purely as a safety net for the case where 0004 and 0005 are + applied as separate deploys (e.g. a squashed or backported migration set) and an + out-of-band write (a fixture load, a data-migration bug elsewhere, a direct DB + edit) creates a mismatched-pair row in the gap between them. + """ + CampaignRun = apps.get_model('solsys_code', 'CampaignRun') + mismatched = CampaignRun.objects.filter( + Q(window_start__isnull=True, window_end__isnull=False) + | Q(window_start__isnull=False, window_end__isnull=True) + ).order_by('pk') + for run in mismatched: + logger.warning( + 'Normalizing mismatched window pair for CampaignRun pk=%s ' + '(window_start=%s, window_end=%s) to fully-TBD before adding ' + 'campaign_run_window_start_end_null_together constraint', + run.pk, + run.window_start, + run.window_end, + ) + run.window_start = None + run.window_end = None + run.save(update_fields=['window_start', 'window_end']) + + +class Migration(migrations.Migration): + dependencies = [ + ('solsys_code', '0004_campaignrun_window_schema'), + ('solsys_code_observatory', '0002_observatory_timezone_seed'), + ('tom_targets', '0030_alter_basetarget_slope'), + ] + + operations = [ + migrations.RunPython(normalize_mismatched_window_pairs, reverse_code=migrations.RunPython.noop), + migrations.AddConstraint( + model_name='campaignrun', + constraint=models.CheckConstraint( + condition=models.Q( + models.Q(('window_end__isnull', True), ('window_start__isnull', True)), + models.Q(('window_end__isnull', False), ('window_start__isnull', False)), + _connector='OR', + ), + name='campaign_run_window_start_end_null_together', + ), + ), + ] diff --git a/solsys_code/migrations/0006_campaignrun_original_obs_date_raw_and_window_needs_review.py b/solsys_code/migrations/0006_campaignrun_original_obs_date_raw_and_window_needs_review.py new file mode 100644 index 00000000..a2cfdae8 --- /dev/null +++ b/solsys_code/migrations/0006_campaignrun_original_obs_date_raw_and_window_needs_review.py @@ -0,0 +1,27 @@ +# Generated by Django 5.2.15 on 2026-07-10 19:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('solsys_code', '0005_campaignrun_campaign_run_window_start_end_null_together'), + ] + + operations = [ + migrations.AddField( + model_name='campaignrun', + name='original_obs_date_raw', + field=models.CharField( + blank=True, default='', max_length=255, verbose_name='Original Obs. Date text (TBD rows only)' + ), + ), + migrations.AddField( + model_name='campaignrun', + name='window_needs_review', + field=models.BooleanField( + default=False, + verbose_name='Whether the observing window could not be automatically resolved and needs manual review', + ), + ), + ] diff --git a/solsys_code/migrations/0007_campaignrun_contact_public_opt_in.py b/solsys_code/migrations/0007_campaignrun_contact_public_opt_in.py new file mode 100644 index 00000000..07e38750 --- /dev/null +++ b/solsys_code/migrations/0007_campaignrun_contact_public_opt_in.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.15 on 2026-07-11 11:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('solsys_code', '0006_campaignrun_original_obs_date_raw_and_window_needs_review'), + ] + + operations = [ + migrations.AddField( + model_name='campaignrun', + name='contact_public_opt_in', + field=models.BooleanField(default=False, verbose_name='Show contact info publicly?'), + ), + ] diff --git a/solsys_code/mixins.py b/solsys_code/mixins.py new file mode 100644 index 00000000..c29ddc1d --- /dev/null +++ b/solsys_code/mixins.py @@ -0,0 +1,11 @@ +from django.contrib.auth.decorators import user_passes_test +from django.utils.decorators import method_decorator + + +class StaffRequiredMixin: + """Redirect to LOGIN_URL unless request.user.is_staff (D-01 approval-queue gate).""" + + @method_decorator(user_passes_test(lambda u: u.is_staff)) + def dispatch(self, *args, **kwargs): + """Redirect non-staff/anonymous requests to LOGIN_URL before dispatching.""" + return super().dispatch(*args, **kwargs) diff --git a/solsys_code/models.py b/solsys_code/models.py index ad364a2e..10c9436a 100644 --- a/solsys_code/models.py +++ b/solsys_code/models.py @@ -1,18 +1,164 @@ -# from django.db import models -# -# from tom_targets.base_models import BaseTarget -# -# -# class UserDefinedTarget(BaseTarget): -# """ -# A target with fields defined by a user. -# """ -# -# class Meta: -# verbose_name = "target" -# permissions = ( -# ('view_target', 'View Target'), -# ('add_target', 'Add Target'), -# ('change_target', 'Change Target'), -# ('delete_target', 'Delete Target'), -# ) +from django.db import models +from tom_calendar.models import CalendarEvent +from tom_targets.models import Target, TargetList + +from solsys_code.solsys_code_observatory.models import Observatory + + +class CalendarEventTelescopeLabel(models.Model): + """Sidecar record of whether a CalendarEvent's telescope label was live-verified + against the LCO API or fallback-guessed (TELESCOPE-03/04). One row per + CalendarEvent at most; no row at all means "verified" by documented default + (e.g. classically-scheduled events from load_telescope_runs, which never go + through telescope-label resolution). + """ + + event = models.OneToOneField( + CalendarEvent, + on_delete=models.CASCADE, + primary_key=True, + related_name='telescope_label_meta', + verbose_name='Calendar event', + ) + is_verified = models.BooleanField( + default=True, verbose_name='Whether the telescope label was live-verified against the LCO API' + ) + + def __str__(self): + return f'{"Verified" if self.is_verified else "Fallback"} label for {self.event.title}' + + +class CampaignRun(models.Model): + """A single target-linked observing run within a coordination campaign (e.g. 3I/ATLAS). + + Replaces the ad-hoc Google Sheet the community previously used to coordinate follow-up + observations of a rare/urgent object. Status is split into two independent fields + (``approval_status``/``run_status``) rather than one flat vocabulary, so a DDT/proposal + request whose real-world outcome is still pending can be represented independently of + admin review state (D-02). The campaign container (``TargetList``) itself carries no + status field in this milestone (D-01) -- status lives entirely on ``CampaignRun``. + """ + + class ApprovalStatus(models.TextChoices): + """Admin review state for a CampaignRun (independent of real-world run outcome).""" + + PENDING_REVIEW = 'pending_review', 'Pending Review' + APPROVED = 'approved', 'Approved' + REJECTED = 'rejected', 'Rejected' + + class RunStatus(models.TextChoices): + """Real-world lifecycle state of a CampaignRun, independent of admin review state.""" + + REQUESTED = 'requested', 'Requested' + PLANNED = 'planned', 'Planned' + OBSERVED = 'observed', 'Observed' + REDUCED = 'reduced', 'Reduced' + PUBLISHED = 'published', 'Published' + CANCELLED = 'cancelled', 'Cancelled' + NOT_AWARDED = 'not_awarded', 'Not Awarded' + WEATHER_TECH_FAILURE = 'weather_tech_failure', 'Weather/Technical Failure' + + campaign = models.ForeignKey( + TargetList, + on_delete=models.PROTECT, + null=False, + related_name='campaign_runs', + verbose_name='Campaign target list', + ) + target = models.ForeignKey( + Target, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name='campaign_runs', + verbose_name='Observed target', + ) + telescope_instrument = models.CharField(max_length=255, verbose_name='Telescope / instrument') + site = models.ForeignKey( + Observatory, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name='campaign_runs', + verbose_name='Resolved observing site', + ) + site_raw = models.CharField(max_length=255, blank=True, default='', verbose_name='Original site code text') + site_needs_review = models.BooleanField( + default=False, verbose_name='Whether the site could not be automatically resolved and needs manual review' + ) + window_start = models.DateField(null=True, blank=True, verbose_name='Observing window start') + window_end = models.DateField(null=True, blank=True, verbose_name='Observing window end') + original_obs_date_raw = models.CharField( + max_length=255, blank=True, default='', verbose_name='Original Obs. Date text (TBD rows only)' + ) + window_needs_review = models.BooleanField( + default=False, + verbose_name='Whether the observing window could not be automatically resolved and needs manual review', + ) + filters_bandpass = models.CharField(max_length=255, blank=True, default='', verbose_name='Filter(s) / bandpass') + observation_details = models.TextField(blank=True, default='', verbose_name='Observation details') + weather = models.TextField(blank=True, default='', verbose_name='Weather conditions or forecast') + observation_outcome = models.TextField(blank=True, default='', verbose_name='Observation outcome') + publication_plans = models.TextField(blank=True, default='', verbose_name='Publication plans') + open_to_collaboration = models.BooleanField(default=False, verbose_name='Open to collaboration?') + comments = models.TextField(blank=True, default='', verbose_name='Other comments') + contact_person = models.CharField(max_length=255, blank=True, default='', verbose_name='Contact person') + contact_email = models.EmailField(blank=True, default='', verbose_name='Contact email') + contact_public_opt_in = models.BooleanField(default=False, verbose_name='Show contact info publicly?') + approval_status = models.CharField( + max_length=20, + choices=ApprovalStatus, + default=ApprovalStatus.PENDING_REVIEW, + verbose_name='Approval status', + ) + run_status = models.CharField( + max_length=30, + choices=RunStatus, + default=RunStatus.REQUESTED, + verbose_name='Run status', + ) + + class Meta: # noqa: D106 + constraints = [ + # WR-05: backs the natural key insert_or_create_campaign_run's docstring and + # import_campaign_csv's D-04 comment both describe as relied on for + # idempotent re-imports. get_or_create() is only race-safe when its lookup + # fields are backed by a real DB constraint; without one, two concurrent + # imports could both miss the existing row and both attempt to create it. + # Resolved-window branch: a concrete single night (window_start == window_end) + # or range. window_end is included (not just window_start) so a range starting + # on the same day as an existing single-night entry is not treated as the same + # row. + models.UniqueConstraint( + fields=('campaign', 'telescope_instrument', 'window_start', 'window_end'), + condition=models.Q(window_start__isnull=False), + name='unique_campaign_run_resolved_window', + ), + # TBD branch: window_start/window_end are deliberately NOT in this constraint's + # field tuple -- they're both NULL for every row this constraint applies to (per + # its own condition), and NULL is never considered equal by a unique constraint + # on any backend, so including them here would silently defeat the whole point + # of this constraint. contact_person is the natural-key discriminator instead + # (never NULL: CharField(blank=True, default='')). + models.UniqueConstraint( + fields=('campaign', 'telescope_instrument', 'contact_person'), + condition=models.Q(window_start__isnull=True), + name='unique_campaign_run_tbd_natural_key', + ), + # WR-02: every reader of window_start/window_end (render_window_start, + # CampaignRunDecisionView.post, claimed_dates) assumes the two fields are either + # both NULL (TBD) or both set (resolved) -- neither partial UniqueConstraint above + # enforces that pairing. Without this, a row with window_start set and + # window_end NULL (or vice versa) would silently persist and crash + # claimed_dates()'s date-arithmetic on read. + models.CheckConstraint( + condition=( + models.Q(window_start__isnull=True, window_end__isnull=True) + | models.Q(window_start__isnull=False, window_end__isnull=False) + ), + name='campaign_run_window_start_end_null_together', + ), + ] + + def __str__(self): + return f'{self.campaign.name}: {self.telescope_instrument} on {self.window_start}' diff --git a/solsys_code/solsys_code_observatory/migrations/0002_observatory_timezone_seed.py b/solsys_code/solsys_code_observatory/migrations/0002_observatory_timezone_seed.py new file mode 100644 index 00000000..1feddcaf --- /dev/null +++ b/solsys_code/solsys_code_observatory/migrations/0002_observatory_timezone_seed.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.19 on 2026-06-12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('solsys_code_observatory', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='observatory', + name='timezone', + field=models.CharField(blank=True, default='', max_length=64, verbose_name='IANA timezone name'), + ), + ] diff --git a/solsys_code/solsys_code_observatory/models.py b/solsys_code/solsys_code_observatory/models.py index 4b706355..139bfcd4 100644 --- a/solsys_code/solsys_code_observatory/models.py +++ b/solsys_code/solsys_code_observatory/models.py @@ -1,9 +1,22 @@ from math import atan2, cos, degrees, radians, sin +import astropy.units as u import erfa +from astropy.coordinates import EarthLocation +from django.core.exceptions import ValidationError from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models -from django.utils import timezone +from django.utils import timezone as django_timezone + +# WR-02 (Phase 22 22-REVIEW.md re-review): campaign_utils.resolve_site()'s tier-3 +# placeholder fallback names a fabricated Observatory f'NEEDS REVIEW: {code}' and +# is_placeholder_observatory() detects any Observatory carrying that exact prefix on its +# `name`. Duplicated here (not imported from solsys_code.campaign_utils, which is this +# module's own single source of truth) to avoid a circular import: campaign_utils imports +# Observatory from this module. Observatory.clean() below rejects the prefix on any +# form-validated save (e.g. the Django admin change form) so a genuine, fully-configured +# Observatory can never be created/renamed to look like a tier-3 placeholder by accident. +NEEDS_REVIEW_NAME_PREFIX = 'NEEDS REVIEW: ' class Observatory(models.Model): @@ -50,6 +63,7 @@ class Observatory(models.Model): db_index=True, ) altitude = models.FloatField(null=True, blank=False, default=0.0, verbose_name='Altitude [m]') + timezone = models.CharField(max_length=64, blank=True, default='', verbose_name='IANA timezone name') observations_type = models.SmallIntegerField( 'Observations Type', null=False, blank=False, default=0, choices=OBSTYPE_CHOICES ) @@ -57,8 +71,8 @@ class Observatory(models.Model): default=False, verbose_name='Whether this observatory uses two-line observations e.g. satellite/radar' ) old_names = models.TextField(blank=True, verbose_name='Any previous names used by the observatory') - created = models.DateTimeField(null=True, blank=False, editable=False, default=timezone.now) - modified = models.DateTimeField(null=True, blank=True, editable=True, default=timezone.now) + created = models.DateTimeField(null=True, blank=False, editable=False, default=django_timezone.now) + modified = models.DateTimeField(null=True, blank=True, editable=True, default=django_timezone.now) # Get Earth's equatorial radius and flattening factor for WGS84 # reference ellipsoid. `r` is in meters @@ -67,6 +81,28 @@ class Observatory(models.Model): def __str__(self) -> str: return f'{self.obscode}: {self.name}' + def clean(self): + """Reject the reserved tier-3-placeholder name prefix on any form-validated save. + + WR-02: ``NEEDS_REVIEW_NAME_PREFIX`` is ``campaign_utils.resolve_site()``'s marker + for a fabricated placeholder Observatory (``campaign_utils.is_placeholder_observatory()`` + checks for it). Without this guard, a staff member creating or renaming a genuine, + fully-configured Observatory to start with that same prefix (e.g. copy-pasting a + Sites Needing Review row's display text) would make the campaign-approval UI treat + it as an eligible-for-replacement placeholder forever -- the inverse of the + never-re-resolve-a-genuine-site invariant that prefix convention exists to protect. + + Only runs on form-validated paths (``full_clean()``, e.g. the Django admin change + form) -- ``resolve_site()``'s own tier-3 fallback creates placeholders via a plain + ``Observatory.objects.create()``, which bypasses ``full_clean()``, so this guard + never blocks the legitimate placeholder-creation path itself. + """ + super().clean() + if self.name and self.name.startswith(NEEDS_REVIEW_NAME_PREFIX): + raise ValidationError( + {'name': f"Observatory name may not start with the reserved '{NEEDS_REVIEW_NAME_PREFIX}' prefix."} + ) + def from_parallax_constants(self, elong: float, rho_cos_phi: float, rho_sin_phi: float): """Convert from MPC parallax constants rho_cos_phi, rho_sin_phi to latitude, altitude and store these @@ -128,6 +164,14 @@ def to_geodetic(self) -> tuple[float, float, float]: """ return (radians(self.lon), radians(self.lat), self.altitude) + def to_earth_location(self) -> EarthLocation: + """Returns the observatory location as an astropy EarthLocation. + + Returns: + EarthLocation: built from this observatory's lon, lat, altitude + """ + return EarthLocation(lon=self.lon * u.deg, lat=self.lat * u.deg, height=self.altitude * u.m) + def ObservatoryXYZ(self) -> tuple[float, float, float]: """Converts the observatory location to geocentric coordinates (in units of Earth radii) Provides similar functionality to Sorcha's Observatory.ObservatoryXYZ() diff --git a/solsys_code/solsys_code_observatory/tests/test_models.py b/solsys_code/solsys_code_observatory/tests/test_models.py index ae8f8dfd..744d08c0 100644 --- a/solsys_code/solsys_code_observatory/tests/test_models.py +++ b/solsys_code/solsys_code_observatory/tests/test_models.py @@ -1,11 +1,12 @@ from datetime import datetime from math import radians +from django.core.exceptions import ValidationError from django.db import IntegrityError from django.test import TestCase # Import models to test -from solsys_code.solsys_code_observatory.models import Observatory +from solsys_code.solsys_code_observatory.models import NEEDS_REVIEW_NAME_PREFIX, Observatory class TestObservatory(TestCase): @@ -21,6 +22,35 @@ def test_creation_noname(self): with self.assertRaises(IntegrityError): bad = Observatory.objects.create(obscode='X05') # noqa: F841 + def test_clean_rejects_reserved_needs_review_name_prefix(self): + """WR-02 (Phase 22 22-REVIEW.md re-review): a form-validated save (full_clean(), e.g. + the Django admin change form) must reject a name starting with the reserved + NEEDS_REVIEW_NAME_PREFIX -- campaign_utils.is_placeholder_observatory()'s marker for + a tier-3 placeholder -- so a genuine Observatory can never be created/renamed to look + like one by accident. Calls clean() directly (not full_clean()) to isolate this + check from the unrelated required-field validation on lat/lon.""" + observatory = Observatory(obscode='X05', name=f'{NEEDS_REVIEW_NAME_PREFIX}Something Real') + with self.assertRaises(ValidationError) as ctx: + observatory.clean() + self.assertIn('name', ctx.exception.message_dict) + + def test_clean_allows_ordinary_name(self): + """A genuine name never sharing the reserved prefix passes clean() unaffected -- + full_clean() still raises for the unrelated required geodetic fields (lat/lon) left + at their null default, proving this isn't a false-positive on unrelated fields.""" + observatory = Observatory(obscode='X05', name='Simonyi Survey Telescope, Rubin Observatory') + try: + observatory.clean() + except ValidationError: + self.fail('clean() must not reject a name without the reserved prefix.') + + def test_tier3_placeholder_create_bypasses_full_clean(self): + """WR-02: resolve_site()'s tier-3 fallback creates placeholders via a plain + Observatory.objects.create() (bypassing full_clean()), so this guard never blocks + the legitimate placeholder-creation path itself.""" + placeholder = Observatory.objects.create(obscode='DCT', name=f'{NEEDS_REVIEW_NAME_PREFIX}DCT', short_name='DCT') + self.assertEqual(placeholder.name, f'{NEEDS_REVIEW_NAME_PREFIX}DCT') + def test_creation_X05(self): expected_parallax_consts = (0.864981, -0.500958) diff --git a/solsys_code/solsys_code_observatory/tests/test_utils.py b/solsys_code/solsys_code_observatory/tests/test_utils.py index b974feb2..1eb3ee06 100644 --- a/solsys_code/solsys_code_observatory/tests/test_utils.py +++ b/solsys_code/solsys_code_observatory/tests/test_utils.py @@ -62,6 +62,30 @@ def test_query_failure_unknown_code(self, mock_get): self.assertEqual(self.bad_code_resp, result) self.assertIsNone(self.fetcher.obs_data) + @patch('requests.get') + def test_query_passes_default_timeout(self, mock_get): + """WR-01: query() must pass an explicit timeout through to requests.get by default.""" + mock_response = MagicMock(ok=True) + mock_response.json.return_value = self.fetcher.obs_data + mock_get.return_value = mock_response + + self.fetcher.query('E10') + + _, kwargs = mock_get.call_args + self.assertIn('timeout', kwargs) + self.assertIsNotNone(kwargs['timeout']) + + @patch('requests.get') + def test_query_passes_explicit_timeout(self, mock_get): + mock_response = MagicMock(ok=True) + mock_response.json.return_value = self.fetcher.obs_data + mock_get.return_value = mock_response + + self.fetcher.query('E10', timeout=3) + + _, kwargs = mock_get.call_args + self.assertEqual(kwargs['timeout'], 3) + def test_to_observatory(self): obs = self.fetcher.to_observatory() @@ -79,6 +103,28 @@ def test_to_observatory(self): self.assertEqual(obs.created, datetime(2019, 5, 25, 0, 11, 26, tzinfo=timezone.utc)) self.assertEqual(obs.modified, datetime(2025, 4, 15, 20, 52, 50, tzinfo=timezone.utc)) + def test_to_observatory_backfills_timezone_from_coordinates(self): + obs = self.fetcher.to_observatory() + + self.assertEqual(obs.timezone, 'Australia/Sydney') + + def test_to_observatory_does_not_clobber_existing_timezone(self): + self.fetcher.obs_data['timezone'] = 'America/New_York' + + obs = self.fetcher.to_observatory() + + self.assertEqual(obs.timezone, 'America/New_York') + + @patch('solsys_code.solsys_code_observatory.utils._get_timezone_finder') + def test_to_observatory_leaves_timezone_blank_when_unresolvable(self, mock_get_finder): + mock_finder = MagicMock() + mock_finder.timezone_at.return_value = None + mock_get_finder.return_value = mock_finder + + obs = self.fetcher.to_observatory() + + self.assertEqual(obs.timezone, '') + def test_to_radar_observatory(self): self.fetcher.obs_data = { 'created_at': 'Sat, 25 May 2019 00:11:21 GMT', diff --git a/solsys_code/solsys_code_observatory/utils.py b/solsys_code/solsys_code_observatory/utils.py index 5e2decdf..76872d1c 100644 --- a/solsys_code/solsys_code_observatory/utils.py +++ b/solsys_code/solsys_code_observatory/utils.py @@ -9,6 +9,28 @@ logger = logging.getLogger(__name__) # logger.setLevel(logging.DEBUG) +_timezone_finder = None + + +def _get_timezone_finder(): + """Return a lazily-constructed, module-cached ``TimezoneFinder`` instance. + + ``TimezoneFinder`` loads its boundary-polygon data on construction, which is + relatively expensive, so it is built once here and reused across every + ``to_observatory()`` call rather than per-call. The import itself is deferred + to inside this function (rather than module level) so that importing + ``utils.py`` -- which happens broadly across the codebase -- doesn't pay the + polygon-data load cost unless a timezone lookup is actually needed. + + :returns: shared ``TimezoneFinder`` instance + """ + global _timezone_finder + if _timezone_finder is None: + from timezonefinder import TimezoneFinder + + _timezone_finder = TimezoneFinder() + return _timezone_finder + class MPCObscodeFetcher: """ @@ -29,7 +51,7 @@ def _flatten_error_dict(self, error_dict): non_field_errors.append(f'{k}: {v}') return non_field_errors - def query(self, obscode: str, dbg: bool = False): + def query(self, obscode: str, dbg: bool = False, timeout: float = 10): """Query the MPC obscodes API for the specific . If successful, the JSON response data is stored in self.obs_data. @@ -37,10 +59,16 @@ def query(self, obscode: str, dbg: bool = False): :type term: str :param dbg: Turns on basic print dump of the key-value pairs (or error response) :type term: bool + :param timeout: request timeout in seconds, passed through to ``requests.get``. + Callers that need "never hang" behavior (e.g. a synchronous per-row import + loop) should rely on this rather than the default of no timeout at all. + :type timeout: float """ self.obs_data = None - response = requests.get('https://data.minorplanetcenter.net/api/obscodes', json={'obscode': obscode}) + response = requests.get( + 'https://data.minorplanetcenter.net/api/obscodes', json={'obscode': obscode}, timeout=timeout + ) if response.ok: self.obs_data = response.json() @@ -55,6 +83,27 @@ def query(self, obscode: str, dbg: bool = False): print('Error: ', response.status_code, self._flatten_error_dict(json_resp)) return json_resp + def query_all(self, timeout: float = 30) -> dict: + """Query the MPC obscodes API for every registered observatory code (bulk mode). + + Omitting the ``obscode`` key from the POST body triggers the bulk-list response + (confirmed live: 2,710 codes, ~1.5 MB, ~1.3s as of 2026-07-11). Stores the result + on ``self.obs_data`` like ``query()`` does, but here it is a dict keyed by 3-char + obscode rather than a single flat observatory dict -- do **not** call + ``to_observatory()`` on a ``query_all()`` result, its ``self.obs_data`` shape + contract is for ``query()`` only. This is a distinct, sibling method: ``query()`` + itself is unmodified. + + :param timeout: request timeout in seconds, passed through to ``requests.get``. + :type timeout: float + :returns: dict keyed by obscode, e.g. {'X09': {'name_utf8': ..., 'longitude': ..., ...}} + :rtype: dict + """ + response = requests.get('https://data.minorplanetcenter.net/api/obscodes', json={}, timeout=timeout) + response.raise_for_status() + self.obs_data = response.json() + return self.obs_data + def to_observatory(self): """ Instantiates a ``Observatory`` object with the data from the obscode query search result. @@ -76,6 +125,17 @@ def to_observatory(self): obs.lon = elong # Convert parallax constants to longitude (again), latitude and altitude obs.from_parallax_constants(elong, float(self.obs_data['rhocosphi']), float(self.obs_data['rhosinphi'])) + # Backfill timezone from the resolved coordinates when the MPC record doesn't + # supply one (it never does in live data). A value already present on the record + # is authoritative and is never overwritten by the coordinate lookup. + obs.timezone = self.obs_data.get('timezone', '') or '' + if not obs.timezone and obs.lat is not None and obs.lon is not None: + tz_name = _get_timezone_finder().timezone_at(lat=obs.lat, lng=obs.lon) + # A coordinate with no timezone polygon (e.g. open ocean) leaves timezone + # blank rather than fabricating a guess, preserving the CR-01 + # resolve-fails-gracefully / stays-retryable behavior. + if tz_name: + obs.timezone = tz_name try: created_time = datetime.strptime(self.obs_data['created_at'], '%a, %d %b %Y %H:%M:%S %Z') created_time = created_time.replace(tzinfo=timezone.utc) diff --git a/solsys_code/solsys_code_observatory/views.py b/solsys_code/solsys_code_observatory/views.py index b699b347..187c57b8 100644 --- a/solsys_code/solsys_code_observatory/views.py +++ b/solsys_code/solsys_code_observatory/views.py @@ -6,6 +6,7 @@ from django.http import HttpResponse from django.shortcuts import redirect from django.urls import reverse_lazy +from django.utils.http import url_has_allowed_host_and_scheme from django.views.generic import CreateView, DetailView, ListView from tom_dataservices.dataservices import MissingDataException @@ -24,11 +25,37 @@ class CreateObservatory(CreateView): template_name = 'solsys_code_observatory/observatory_create.html' def get_success_url(self): - """Create a custom success_url to redirect to the detail page for the - newly created Observatory. + """Redirect to a validated ``?next=`` target (SITE-02/D-05) when present -- e.g. back + to the approval queue for the "Create new Observatory" round-trip from Plan 21-03 -- + falling back to the detail page for the newly created Observatory otherwise. + Validated with ``url_has_allowed_host_and_scheme`` so an off-host/bad-scheme ``next`` + can never be used as an open redirect (T-21-06). """ + next_url = self.request.GET.get('next') or self.request.POST.get('next') + if next_url and url_has_allowed_host_and_scheme( + next_url, allowed_hosts={self.request.get_host()}, require_https=self.request.is_secure() + ): + return next_url return reverse_lazy('solsys_code_observatory:detail', kwargs={'pk': self.kwargs['pk']}) + def get_initial(self): + """Pre-fill the ``obscode`` field from ``?obscode=`` (SITE-02/D-05) -- e.g. the typed + text from an unresolved approval-queue row via Plan 21-03's "Create new Observatory" + link -- so staff don't have to retype it. + """ + initial = super().get_initial() + raw_obscode = self.request.GET.get('obscode', '') + # WR-02: ``?obscode=`` may carry an unresolved approval-queue row's ``site_raw`` + # verbatim (campaign_tables.py's "Create new Observatory" link), which is frequently + # a full site name rather than a real obscode. CreateObservatoryForm.obscode is + # exactly 3 characters (min_length=max_length=3), so pre-filling anything else is + # guaranteed invalid on first render -- only pre-fill when the raw value plausibly + # is an obscode; otherwise leave the field blank so it doesn't look "already filled + # in" with a value staff must notice and fully overwrite. + if len(raw_obscode) == 3: + initial['obscode'] = raw_obscode + return initial + def get_context_data(self, **kwargs): # noqa: D102 context = super().get_context_data(**kwargs) return context diff --git a/solsys_code/telescope_runs.py b/solsys_code/telescope_runs.py new file mode 100644 index 00000000..67b280e3 --- /dev/null +++ b/solsys_code/telescope_runs.py @@ -0,0 +1,497 @@ +import re +from dataclasses import dataclass +from datetime import date as date_cls +from datetime import datetime, time +from math import sqrt +from zoneinfo import ZoneInfo + +import astropy.units as u +import numpy as np +from astropy.coordinates import AltAz, get_sun +from astropy.time import Time + +from solsys_code.solsys_code_observatory.models import Observatory + +# Maps telescope name to MPC observatory code. `Observatory` (looked up via +# get_site()) remains the single source of truth for location and timezone. +SITES = { + 'Magellan-Clay': '268', + 'Magellan-Baade': '269', + 'NTT': '809', + 'FTS': 'E10', +} + +# Sites whose classical-run date ranges follow ESO's noon-to-noon convention. +# For these sites (e.g. NTT / La Silla) the date range is transcribed verbatim +# from ESO's Tatoo scheduling tool, whose displayed END date is the noon-to-noon +# *closing boundary* of the last night, NOT itself an observing night -- so the +# last observing night is day2 - 1 (E - S nights). Las Campanas (Magellan) sites, +# by contrast, treat Start and End as BOTH inclusive observing nights (E - S + 1 +# nights); see docs/design/telescope_runs_calendar.rst "Night convention". This +# distinction is applied in load_telescope_runs._iter_run_nights(). +ESO_NOON_TO_NOON_SITES = frozenset({'NTT'}) + +# Known classical-schedule status words/phrases (case-insensitive), per +# docs/design/telescope_runs_calendar.rst "Classical Run Input Format". +KNOWN_STATUSES = {'allocation', 'proposed', 'confirmed', 'cancelled', 'not confirmed'} + +# Full month names and 3-letter abbreviations, case-insensitive, mapped to 1-12. +_MONTH_NAMES = { + 'jan': 1, + 'january': 1, + 'feb': 2, + 'february': 2, + 'mar': 3, + 'march': 3, + 'apr': 4, + 'april': 4, + 'may': 5, + 'jun': 6, + 'june': 6, + 'jul': 7, + 'july': 7, + 'aug': 8, + 'august': 8, + 'sep': 9, + 'september': 9, + 'oct': 10, + 'october': 10, + 'nov': 11, + 'november': 11, + 'dec': 12, + 'december': 12, +} + +_MONTH_NAME_PATTERN = '|'.join(sorted(_MONTH_NAMES, key=len, reverse=True)) + +# month-after-range, e.g. 'Jul 8-12' +_MONTH_AFTER_RANGE = re.compile( + rf""" + (?P{_MONTH_NAME_PATTERN})\s+ + (?P\d{{1,2}}) + \s*-\s* + (?P\d{{1,2}}) + """, + re.VERBOSE | re.IGNORECASE, +) + +# month-before-range, e.g. '9-13 July' +_MONTH_BEFORE_RANGE = re.compile( + rf""" + (?P\d{{1,2}}) + \s*-\s* + (?P\d{{1,2}}) + \s+ + (?P{_MONTH_NAME_PATTERN}) + """, + re.VERBOSE | re.IGNORECASE, +) + +# cross-month range, e.g. '28 December-2 January' +_CROSS_MONTH_RANGE = re.compile( + rf""" + (?P\d{{1,2}})\s+ + (?P{_MONTH_NAME_PATTERN}) + \s*-\s* + (?P\d{{1,2}})\s+ + (?P{_MONTH_NAME_PATTERN}) + """, + re.VERBOSE | re.IGNORECASE, +) + +# Status as a parenthesized phrase, e.g. '(proposed)' or '(not confirmed)'. +_PAREN_STATUS = re.compile(r'\(([^)]+)\)') + +# Partial nights matcher +_PARTIAL_NIGHTS = re.compile( + r""" + (BoN|\d{4})-(EoN|\d{4}) + """, + re.VERBOSE | re.IGNORECASE, +) + + +def get_site(name: str) -> Observatory: + """Resolves a telescope name to its Observatory record. + + Args: + name: Telescope name, a key of SITES (e.g. 'Magellan-Clay'). + + Returns: + Observatory: the observatory record for this telescope's site. + + Raises: + Observatory.DoesNotExist: if name is not a key in SITES, or no + Observatory record exists for the resolved MPC obscode. + """ + try: + obscode = SITES[name] + except KeyError as exc: + raise Observatory.DoesNotExist(f'No site registered in SITES for telescope {name!r}') from exc + return Observatory.objects.get(obscode=obscode) + + +def horizon_dip(altitude_m: float) -> u.Quantity: + """Horizon dip for an observer at altitude_m metres. + + dip = 1.76 arcmin * sqrt(altitude in metres). + + This is the Nautical Almanac dip formula (terrestrial refraction k~1/6 + folded into the spherical-geometry estimate dip ~ sqrt(2h/R), R=6371 km); + see docs/design/telescope_runs_calendar.rst ("Astronomy: Night + Boundaries") for the derivation. + + The sqrt(2h/R) model only describes the depression of the visible horizon + for an observer *elevated above* the reference surface (h > 0). At or below + sea level there is no such depression, so the dip is 0. A small negative + altitude is physically normal for real, near-sea-level MPC observatories: + the MPC publishes parallax constants to only 5 decimal places (~64 m of + altitude granularity, see Observatory.from_parallax_constants), so a genuine + near-sea-level site — e.g. obscode 434 "S. Benedetto Po" at ~19 m real + elevation — round-trips to a small negative geodetic height (-18.83 m). + Rejecting those would strand the site's calendar projection, so any + altitude <= 0 is treated as sea level (dip = 0). + + Args: + altitude_m: Observer altitude above sea level, in metres. Values <= 0 + (at or below sea level) yield a 0 dip. + + Returns: + u.Quantity: dip angle (e.g. 1.4376 deg for 2402 m; 0 arcmin at or below + sea level). + + Raises: + ValueError: if altitude_m is None (an unset altitude is a data error, + not a physical location). + """ + if altitude_m is None: + raise ValueError(f'altitude_m must be a number, got {altitude_m!r}') + # Clamp at/below-sea-level altitudes to a 0 dip: the sqrt(2h/R) model has no + # elevated horizon to depress there, and this keeps near-sea-level MPC sites + # (small negative parallax-derived heights) projectable rather than crashing. + if altitude_m <= 0: + return 0.0 * u.arcmin + return 1.76 * sqrt(altitude_m) * u.arcmin + + +def _solar_altitude(times: Time, location) -> np.ndarray: + """Solar altitude in degrees for an array of Time objects at a location. + + Args: + times: astropy Time array of evaluation epochs. + location: astropy EarthLocation of the observer. + + Returns: + np.ndarray: solar altitude in degrees for each time. + """ + sun = get_sun(times) + altaz = sun.transform_to(AltAz(obstime=times, location=location)) + return altaz.alt.deg + + +def _find_crossing( + anchor: Time, location, threshold_deg: float, search_hours: float = 24, coarse_step_min: float = 1.0 +) -> list[Time]: + """Finds UTC times where solar altitude crosses threshold_deg. + + Performs a coarse scan over the window [anchor, anchor + search_hours], + then refines each sign change with bisection to sub-second precision. + Anchoring at local noon of the observing date (see _local_noon_utc) and + scanning forward search_hours=24 guarantees both the evening sunset/dark + crossing of that date and the following morning's sunrise/dark-end + crossing fall within the window, in chronological (set, then rise) order. + + Args: + anchor: astropy Time at the start of the search window (local noon). + location: astropy EarthLocation of the observer. + threshold_deg: solar altitude threshold to find crossings of, in degrees. + search_hours: total width of the search window, in hours. + coarse_step_min: coarse scan step size, in minutes. + + Returns: + list[Time]: UTC times of each altitude crossing, in chronological order. + """ + # +coarse_step_min so the window covers a full, closed [0, search_hours] range + # (np.arange's exclusive upper bound would otherwise leave the last minute unscanned). + offsets = np.arange(0, search_hours * 60 + coarse_step_min, coarse_step_min) * u.min + times = anchor + offsets + alt = _solar_altitude(times, location) + crossings = [] + for i in range(len(alt) - 1): + if (alt[i] - threshold_deg) * (alt[i + 1] - threshold_deg) < 0: + # Bisection refine between times[i] and times[i+1] + lo, hi = times[i], times[i + 1] + lo_alt = alt[i] + for _ in range(10): # ~1/1024 of 1-min step -> sub-second precision + mid = lo + (hi - lo) / 2 + mid_alt = _solar_altitude(Time([mid]), location)[0] + if (mid_alt - threshold_deg) * (lo_alt - threshold_deg) < 0: + hi = mid + else: + lo, lo_alt = mid, mid_alt + crossings.append(lo) + return crossings + + +def _local_noon_utc(local_date: date_cls, tz_name: str) -> Time: + """Local noon of local_date, converted to UTC, as an astropy Time. + + Args: + local_date: the local calendar date. + tz_name: IANA timezone name for the site (e.g. 'America/Santiago'). + + Returns: + Time: local noon of local_date, expressed as a UTC astropy Time. + """ + tz = ZoneInfo(tz_name) + local_noon = datetime.combine(local_date, time(12, 0), tzinfo=tz) + return Time(local_noon.astimezone(ZoneInfo('UTC'))) + + +def sun_event(site: Observatory, date: date_cls, kind: str) -> tuple[Time, Time]: + """Computes UTC sun-event crossing times for an observing night. + + Args: + site: Observatory instance (from get_site()). + date: local calendar date of sunset; the returned events cover the + observing night starting on the evening of this date. + kind: 'sun' for the dip-corrected sunset/sunrise threshold + (-(0.833 + dip) degrees), or 'dark' for the -15 degree + dark-window threshold (no dip correction). + + Returns: + tuple[Time, Time]: (setting, rising) as astropy.time.Time objects, + UTC scale. + + Raises: + ValueError: if kind is not 'sun' or 'dark'; if site.timezone is + unset; or if the solar altitude does not cross threshold + exactly twice in the 24h window following local noon (e.g. a + high-latitude site in summer where the sun never sets, or never + gets dark). + """ + if not site.timezone: + raise ValueError( + f'Observatory {site.short_name!r} (obscode={site.obscode}) has no timezone set; ' + 'set Observatory.timezone (IANA name, e.g. "America/Santiago") before calling sun_event().' + ) + + location = site.to_earth_location() + anchor = _local_noon_utc(date, site.timezone) + + if kind == 'sun': + dip = horizon_dip(site.altitude) + # 0.833 deg = standard solar semi-diameter (~16') + horizon refraction (~34') + threshold = -(0.833 + dip.to_value(u.deg)) + elif kind == 'dark': + threshold = -15.0 + else: + raise ValueError(f"kind must be 'sun' or 'dark', got {kind!r}") + + crossings = _find_crossing(anchor, location, threshold, search_hours=24) + if len(crossings) != 2: + raise ValueError( + f'Expected 2 sun-event crossings for {site.short_name} on {date} ' + f'(kind={kind!r}), got {len(crossings)}: {crossings}. ' + 'This can happen at high latitudes when the sun never sets or never ' + 'reaches the requested threshold (e.g. midnight sun or no astronomical darkness).' + ) + return crossings[0], crossings[1] + + +@dataclass(frozen=True) +class ParsedRun: + """Structured result of parse_run_line(). + + Attributes: + telescope: resolved SITES key (e.g. 'NTT'). + instrument: instrument name as it appears in the run line (may be + hyphenated, e.g. 'Proto-Lightspeed'). + status: lowercase status word/phrase, e.g. 'allocation', 'proposed', + 'not confirmed'. Defaults to 'allocation' if absent (D-05). + year: four-digit year. Defaults to the current year (PARSE-03), or + current year + 1 for a run that starts in December and ends in + January (year roll-over). + month: month number (1-12) of day1 (the start of the run). + day1: first day of the run (inclusive). + day2: last day of the run (inclusive). + start_window: optional start-of-window token — 'BoN' (computed sunset) + or a 4-digit HHMM UTC string. Times < 1200 are on d+1 morning; + times >= 1200 are on d evening. None means full night from sunset. + end_window: optional end-of-window token — 'EoN' (computed sunrise) + or a 4-digit HHMM UTC string. None means full night to sunrise. + """ + + telescope: str + instrument: str + status: str + year: int + month: int + day1: int + day2: int + start_window: str | None = None + end_window: str | None = None + + +def _resolve_telescope(token: str) -> str: + """Resolves a telescope token to a SITES key by prefix match (D-01). + + Args: + token: the first whitespace-delimited token of a run line. + + Returns: + str: the resolved SITES key. + + Raises: + ValueError: if token is a prefix of zero or 2+ SITES keys. + """ + if token in SITES: + return token + candidates = [key for key in SITES if key.startswith(token)] + if len(candidates) == 1: + return candidates[0] + if len(candidates) > 1: + raise ValueError( + f'Ambiguous telescope {token!r}: matches multiple SITES keys {candidates}; ' + 'use a more specific telescope name (e.g. "Magellan-Clay" or "Magellan-Baade").' + ) + raise ValueError(f'Unknown telescope {token!r}: does not match any SITES key {list(SITES)}') + + +def _resolve_status(line: str) -> tuple[str, str]: + """Extracts and validates the status word/phrase from a run line (D-04/05/06). + + Args: + line: the full run line (including any parenthesized status). + + Returns: + tuple[str, str]: (status, remainder) where status is the lowercase + KNOWN_STATUSES member (defaulting to 'allocation' if absent) and + remainder is the line with the status token(s) removed. + + Raises: + ValueError: if a parenthesized phrase or trailing status-shaped word + is present but not in KNOWN_STATUSES. + """ + paren_match = _PAREN_STATUS.search(line) + if paren_match: + candidate = paren_match.group(1).strip().lower() + if candidate not in KNOWN_STATUSES: + raise ValueError( + f'Unrecognized status {candidate!r} in {line!r}; known statuses are {sorted(KNOWN_STATUSES)}' + ) + remainder = line[: paren_match.start()] + line[paren_match.end() :] + return candidate, remainder + + # Multi-word statuses (e.g. 'not confirmed') checked before single words. + for status in sorted(KNOWN_STATUSES, key=len, reverse=True): + match = re.search(rf'(? ParsedRun: + """Parses a free-text classical-schedule run line into structured fields. + + Expected format (per docs/design/telescope_runs_calendar.rst "Classical + Run Input Format"): ``telescope instrument [status] daterange [(status)]``, + e.g. 'NTT EFOSC2 allocation 9-13 July' or 'Magellan Proto-Lightspeed Jul + 8-12 (proposed)'. The date range may have the month name before or after + the day range, and no year is given (year defaults per PARSE-03). + + An optional trailing window token of the form ``(BoN|HHMM)-(EoN|HHMM)`` + restricts the event to a portion of the night, e.g. 'BoN-0626' or + '0646-EoN'. HHMM < 1200 is treated as next-morning UTC; HHMM >= 1200 is + same-evening UTC. + + Args: + line: a single run-line string. + + Returns: + ParsedRun: the parsed telescope, instrument, status, year, month, + day1, day2, and optional start_window/end_window. + + Raises: + ValueError: if line is empty, the telescope token does not resolve to + exactly one SITES key (D-01), the status is unrecognized (D-06), + no date range can be found, or the trailing window token is present + but malformed. + """ + stripped = line.strip() + if not stripped: + raise ValueError('parse_run_line() received an empty line') + + status, remainder = _resolve_status(stripped) + + # Date range: try month-after-range ('Jul 8-12'), cross-month + # ('28 December-2 January'), then month-before-range ('9-13 July'). + match = _MONTH_AFTER_RANGE.search(remainder) + if match: + day1 = int(match.group('day1')) + day2 = int(match.group('day2')) + month = _MONTH_NAMES[match.group('month1').lower()] + else: + match = _CROSS_MONTH_RANGE.search(remainder) + if match: + day1 = int(match.group('day1')) + day2 = int(match.group('day2')) + month = _MONTH_NAMES[match.group('month1').lower()] + else: + match = _MONTH_BEFORE_RANGE.search(remainder) + if not match: + raise ValueError(f'Could not find a date range (e.g. "9-13 July" or "Jul 8-12") in {line!r}') + day1 = int(match.group('day1')) + day2 = int(match.group('day2')) + month = _MONTH_NAMES[match.group('month1').lower()] + + # Year (PARSE-03): default to current year; roll over to next year if the + # run starts in December and ends in January (cross-year range). + year = date_cls.today().year + if month == 12 and day2 < day1: + year += 1 + + # Telescope (token 0) and instrument (token 1, possibly hyphenated). + before_range = remainder[: match.start()] + tokens = before_range.split() + if len(tokens) < 2: + raise ValueError(f'Could not find telescope and instrument tokens in {line!r}') + telescope_token, instrument = tokens[0], tokens[1] + + # D-06: any remaining word(s) between instrument and the date range are a + # status-shaped token that must be in KNOWN_STATUSES (already checked and + # consumed by _resolve_status if recognized). + leftover = ' '.join(tokens[2:]).strip() + if leftover: + raise ValueError(f'Unrecognized status {leftover!r} in {line!r}; known statuses are {sorted(KNOWN_STATUSES)}') + + telescope = _resolve_telescope(telescope_token) + + # Optional trailing window token restricts the event to a portion of the + # night, e.g. 'BoN-0626' or '0646-EoN'. + after_range = remainder[match.end() :] + window_tokens = after_range.split() + start_window = end_window = None + if len(window_tokens) == 1: + window_match = _PARTIAL_NIGHTS.search(window_tokens[0]) + if window_match: + start_window = window_match.group(1) + end_window = window_match.group(2) + else: + raise ValueError(f'Unrecognized partial night token {after_range.strip()!r} in {line!r}') + elif len(window_tokens) > 1: + raise ValueError(f'Unexpected trailing tokens {after_range.strip()!r} in {line!r}') + + return ParsedRun( + telescope=telescope, + instrument=instrument, + status=status, + year=year, + month=month, + day1=day1, + day2=day2, + start_window=start_window, + end_window=end_window, + ) diff --git a/solsys_code/templatetags/__init__.py b/solsys_code/templatetags/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/solsys_code/templatetags/calendar_display_extras.py b/solsys_code/templatetags/calendar_display_extras.py new file mode 100644 index 00000000..c00623f5 --- /dev/null +++ b/solsys_code/templatetags/calendar_display_extras.py @@ -0,0 +1,198 @@ +"""Django template tag library for proposal color and status visual encoding. + +Provides three simple_tags consumed by calendar.html (Plan 02): + +- proposal_color: deterministic, colorblind-vetted palette color keyed by proposal code (DISPLAY-04) +- status_border_css: title-prefix → box-shadow CSS fragment (DISPLAY-06) +- visible_proposals: current-month legend data grouped by color (DISPLAY-07) + +All values returned by proposal_color and status_border_css are drawn from fixed +internal constants — the raw proposal/title string is used only as a hash input or +startswith test and is never echoed into the output (T-09-01/T-09-02 mitigations). +""" + +import hashlib +from collections import defaultdict + +from django import template + +register = template.Library() + +# Colorblind-vetted, white-text-AA palette — 8 hex values locked by 09-UI-SPEC.md +# Color section. Mutual distinguishability verified against CVD simulators for +# deuteranopia + protanopia (see 09-VALIDATION.md manual verification item A1). +PROPOSAL_PALETTE = [ + '#005f9e', + '#a34000', + '#5b2080', + '#006b4e', + '#9e1c1c', + '#006b6b', + '#6b2060', + '#7a4500', +] + +# D-05: dedicated neutral slot for calendar events with no proposal code. +# Separate from PROPOSAL_PALETTE so an empty-string hash cannot accidentally +# collide with this value (see 09-RESEARCH Pitfall 1). +NEUTRAL_SLOT_COLOR = '#5a6268' + +# D-06: human-readable label for classical-schedule (empty-proposal) legend entry. +CLASSICAL_SCHEDULE_LABEL = 'Classical schedule' + +# Title-prefix vocabulary emitted by sync_lco_observation_calendar.py (confirmed live), plus +# '[WEATHERED]' (D-03, campaign_views._RUN_STATUS_CALENDAR_PREFIX, Phase 23 Plan 02) -- +# both must stay byte-identical. Terminal states: observations that reached an +# unrecoverable failure state. [QUEUED] is handled separately (its own branch below). +_TERMINAL_PREFIXES = ('[EXPIRED]', '[CANCELLED]', '[FAILED]', '[WEATHERED]') + + +@register.simple_tag +def proposal_color(proposal: str) -> str: + """Return a deterministic hex color for a proposal code (DISPLAY-04). + + Normalizes via .strip().upper() before hashing so casing and whitespace + variants share one color — D-04 premise, 09-RESEARCH Pitfall 1. Uses + hashlib.sha256 for deterministic output across process restarts (see + STATE.md Key Technical Notes — the per-process-salted built-in is forbidden + here). + + Args: + proposal: Raw proposal string from CalendarEvent.proposal (may be + blank, mixed-case, or have surrounding whitespace). + + Returns: + A hex color string from PROPOSAL_PALETTE, or NEUTRAL_SLOT_COLOR for + blank/missing proposals (D-05). + """ + normalized = (proposal or '').strip().upper() + if not normalized: + return NEUTRAL_SLOT_COLOR + digest = hashlib.sha256(normalized.encode()).hexdigest() + return PROPOSAL_PALETTE[int(digest, 16) % len(PROPOSAL_PALETTE)] + + +@register.simple_tag +def status_border_css(title: str) -> str: + """Return a CSS box-shadow fragment encoding the observation status (DISPLAY-06). + + Maps the title-prefix vocabulary from sync_lco_observation_calendar.py to a + box-shadow ring (D-08 resolved=box-shadow). The placed bucket ([UNVERIFIED] + or no prefix) intentionally returns '' because Phase 8's D-09-reserved + border treatment already owns the verified/fallback visual distinction — + re-encoding it here would cause the two signals to merge into one style + attribute branch instead of composing independently (09-RESEARCH Pitfall 3 + prevention). + + Args: + title: CalendarEvent.title — may start with a known status prefix. + + Returns: + A CSS fragment suitable for direct inclusion in a style attribute, e.g. + 'box-shadow: 0 0 0 2px rgba(0, 0, 0, 0.45);'. Returns '' for placed + events. The D-09-reserved border style is never emitted by this tag. + """ + title = title or '' + if title.startswith('[QUEUED] '): + return 'box-shadow: 0 0 0 2px rgba(0, 0, 0, 0.45);' + if any(title.startswith(p) for p in _TERMINAL_PREFIXES): + return 'box-shadow: 0 0 0 3px rgba(160, 0, 0, 0.55);' + return '' + + +def _relative_luminance(hex_color: str) -> float: + """Return relative luminance (0.0–1.0) for a #rrggbb hex color per WCAG 2.1.""" + if not hex_color or not isinstance(hex_color, str): + return 0.0 # treat invalid input as black (worst case → white text returned) + h = hex_color.lstrip('#') + if len(h) != 6: + return 0.0 + r, g, b = int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16) + + def linearize(c: int) -> float: + L = c / 255 + return L / 12.92 if L <= 0.04045 else ((L + 0.055) / 1.055) ** 2.4 + + return 0.2126 * linearize(r) + 0.7152 * linearize(g) + 0.0722 * linearize(b) + + +@register.simple_tag +def text_color_for_bg(hex_color: str) -> str: + """Return '#fff' or '#000' — whichever achieves WCAG AA 4.5:1 contrast against hex_color (DISPLAY-08). + + Uses the WCAG 2.1 relative luminance formula. White text achieves 4.5:1 against + any background with luminance <= 0.183; all PROPOSAL_PALETTE and NEUTRAL_SLOT_COLOR + entries are dark, so '#fff' is returned for all current palette members. + + Args: + hex_color: A '#rrggbb' hex color string (e.g. '#005f9e'). + + Returns: + '#fff' if white text achieves >= 4.5:1 contrast; '#000' otherwise. + """ + lum = _relative_luminance(hex_color) + white_contrast = 1.05 / (lum + 0.05) + return '#fff' if white_contrast >= 4.5 else '#000' + + +@register.simple_tag +def visible_proposals(weeks) -> list[dict]: + """Compute the set of proposals visible in the currently-rendered month (DISPLAY-07). + + Iterates the weeks/day context already materialized by render_calendar() — + no new database query (D-02). Groups by resulting color so hash-colliding + proposals share one legend entry (D-04, 09-RESEARCH Pitfall 4). Neutral-slot + events (empty proposal) appear as 'Classical schedule' and are forced last + regardless of their hex sort position (D-06 / 09-UI-SPEC.md Legend Layout). + + Args: + weeks: The weeks context list passed to calendar.html — a list of lists + of day objects, each with .all_day_events and .events attributes + containing objects with a .proposal attribute. + + Returns: + List of dicts with keys 'color' (hex string), 'codes' (sorted list of + proposal code strings or [CLASSICAL_SCHEDULE_LABEL] for the neutral + slot), and 'label' (comma-joined string for display). Sorted by color + hex ascending, with the NEUTRAL_SLOT_COLOR entry appended last. + """ + by_color: dict[str, set[str]] = defaultdict(set) + for week in weeks: + for day in week: + # Support both dict-based days (tom_calendar view) and attribute-based + # stubs (unit tests using SimpleNamespace or similar objects). + if isinstance(day, dict): + all_day = day['all_day_events'] + timed = day['events'] + else: + all_day = day.all_day_events + timed = day.events + for event in list(all_day) + list(timed): + normalized = (event.proposal or '').strip().upper() + color = proposal_color(event.proposal) + label = normalized if normalized else CLASSICAL_SCHEDULE_LABEL + by_color[color].add(label) + + result = [] + for color, codes in sorted(by_color.items()): + if color == NEUTRAL_SLOT_COLOR: + continue + result.append( + { + 'color': color, + 'codes': sorted(codes), + 'label': ', '.join(sorted(codes)), + } + ) + + if NEUTRAL_SLOT_COLOR in by_color: + codes = by_color[NEUTRAL_SLOT_COLOR] + result.append( + { + 'color': NEUTRAL_SLOT_COLOR, + 'codes': sorted(codes), + 'label': ', '.join(sorted(codes)), + } + ) + + return result diff --git a/solsys_code/tests/test_admin.py b/solsys_code/tests/test_admin.py new file mode 100644 index 00000000..a45f41f2 --- /dev/null +++ b/solsys_code/tests/test_admin.py @@ -0,0 +1,76 @@ +"""Tests for solsys_code/admin.py -- proves the load-bearing admin constraints via the +admin test client rather than by eyeballing the ModelAdmin class definitions: + +- CampaignRun and CalendarEventTelescopeLabel are both reachable under /admin/solsys_code/. +- approval_status is visible-but-non-editable in the CampaignRun change form (T-jpd-01: no + admin path to APPROVED that bypasses CampaignRunDecisionView.post()'s calendar projection + + D-06 clobber guard). +- contact_person/contact_email never appear in the CampaignRun change-list (T-jpd-02: PII is + not scannable across rows) but remain editable in the detail/change view. +- CalendarEventTelescopeLabel's event__title search path resolves without a FieldError. +""" + +from django.contrib.auth.models import User +from django.test import TestCase +from django.urls import reverse +from tom_targets.models import TargetList + +from solsys_code.models import CampaignRun + +PII_CONTACT_PERSON = 'Zztestcontact' +PII_CONTACT_EMAIL = 'pii-secret@example.test' + + +class AdminRegistrationAndGatingTests(TestCase): + """T-jpd-01/T-jpd-02: approval_status read-only, PII gated from the change-list.""" + + @classmethod + def setUpTestData(cls) -> None: + cls.superuser = User.objects.create_superuser(username='adminuser', email='admin@example.test', password='pw') + cls.campaign = TargetList.objects.create(name='3I/ATLAS') + # NOTE: not named `cls.run` -- unittest.TestCase.run() is the method the test + # framework itself invokes to execute each test; shadowing it with an attribute + # breaks test execution with `TypeError: 'CampaignRun' object is not callable`. + cls.campaign_run = CampaignRun.objects.create( + campaign=cls.campaign, + telescope_instrument='LCO-1m-Sinistro', + contact_person=PII_CONTACT_PERSON, + contact_email=PII_CONTACT_EMAIL, + ) + + def setUp(self) -> None: + self.client.force_login(self.superuser) + + def test_campaignrun_changelist_loads(self) -> None: + response = self.client.get(reverse('admin:solsys_code_campaignrun_changelist')) + self.assertEqual(response.status_code, 200) + + def test_calendareventtelescopelabel_changelist_loads(self) -> None: + response = self.client.get(reverse('admin:solsys_code_calendareventtelescopelabel_changelist')) + self.assertEqual(response.status_code, 200) + + def test_calendareventtelescopelabel_search_resolves(self) -> None: + response = self.client.get( + reverse('admin:solsys_code_calendareventtelescopelabel_changelist'), {'q': 'anything'} + ) + self.assertEqual(response.status_code, 200) + + def test_approval_status_is_readonly_in_change_form(self) -> None: + response = self.client.get(reverse('admin:solsys_code_campaignrun_change', args=[self.campaign_run.pk])) + self.assertEqual(response.status_code, 200) + content = response.content.decode() + self.assertIn('Pending Review', content) + self.assertNotIn('name="approval_status"', content) + + def test_contact_fields_editable_in_change_form(self) -> None: + response = self.client.get(reverse('admin:solsys_code_campaignrun_change', args=[self.campaign_run.pk])) + content = response.content.decode() + self.assertIn('name="contact_person"', content) + self.assertIn('name="contact_email"', content) + + def test_pii_not_rendered_in_changelist(self) -> None: + response = self.client.get(reverse('admin:solsys_code_campaignrun_changelist')) + content = response.content.decode() + self.assertNotIn(PII_CONTACT_PERSON, content) + self.assertNotIn(PII_CONTACT_EMAIL, content) + self.assertIn('LCO-1m-Sinistro', content) diff --git a/solsys_code/tests/test_calendar_display_extras.py b/solsys_code/tests/test_calendar_display_extras.py new file mode 100644 index 00000000..c49b3d1e --- /dev/null +++ b/solsys_code/tests/test_calendar_display_extras.py @@ -0,0 +1,189 @@ +"""Unit tests for solsys_code.templatetags.calendar_display_extras. + +Wave 0 scaffold — written before the module exists (RED). Tests cover the three +public tags: proposal_color (DISPLAY-04, D-04/D-05), status_border_css (DISPLAY-06, +D-08/D-09), and visible_proposals (DISPLAY-07, D-02/D-04/D-06). +""" + +from types import SimpleNamespace + +from django.test import TestCase + +from solsys_code.templatetags.calendar_display_extras import ( + CLASSICAL_SCHEDULE_LABEL, + NEUTRAL_SLOT_COLOR, + PROPOSAL_PALETTE, + proposal_color, + status_border_css, + text_color_for_bg, + visible_proposals, +) + +QUEUED_BOX_SHADOW = 'box-shadow: 0 0 0 2px rgba(0, 0, 0, 0.45);' +TERMINAL_BOX_SHADOW = 'box-shadow: 0 0 0 3px rgba(160, 0, 0, 0.55);' + + +class ProposalColorTest(TestCase): + def test_same_input_same_output(self): + # DISPLAY-04: deterministic — same proposal always returns the same color. + self.assertEqual(proposal_color('LTP2025A-004'), proposal_color('LTP2025A-004')) + + def test_normalization_case_insensitive(self): + # D-04 premise: .strip().upper() applied before hashing. + self.assertEqual(proposal_color('LTP2025A-004'), proposal_color('ltp2025a-004')) + + def test_normalization_trailing_space(self): + # D-04 premise: whitespace stripped before hashing. + self.assertEqual(proposal_color('LTP2025A-004'), proposal_color('LTP2025A-004 ')) + + def test_empty_string_returns_neutral_slot(self): + # D-05: empty proposal → dedicated neutral slot, not hash-of-empty. + self.assertEqual(proposal_color(''), NEUTRAL_SLOT_COLOR) + + def test_blank_string_returns_neutral_slot(self): + # D-05: whitespace-only proposal → neutral slot after .strip(). + self.assertEqual(proposal_color(' '), NEUTRAL_SLOT_COLOR) + + def test_none_returns_neutral_slot(self): + # D-05: None proposal → neutral slot. + self.assertEqual(proposal_color(None), NEUTRAL_SLOT_COLOR) + + def test_nonempty_proposal_returns_palette_member(self): + # D-04: non-empty proposals map to one of the 8 curated palette entries. + color = proposal_color('LTP2025A-004') + self.assertIn(color, PROPOSAL_PALETTE) + + def test_neutral_slot_not_in_palette(self): + # D-05: neutral slot is a separate slot — not a palette hash target. + self.assertNotIn(NEUTRAL_SLOT_COLOR, PROPOSAL_PALETTE) + + +class StatusBorderCssTest(TestCase): + def test_queued_returns_queued_box_shadow(self): + # D-08: [QUEUED]-prefixed title → queued ring. + result = status_border_css('[QUEUED] LTP run') + self.assertEqual(result, QUEUED_BOX_SHADOW) + + def test_expired_returns_terminal_box_shadow(self): + # D-08: [EXPIRED]-prefixed title → terminal-failure ring. + self.assertEqual(status_border_css('[EXPIRED] x'), TERMINAL_BOX_SHADOW) + + def test_cancelled_returns_terminal_box_shadow(self): + # D-08: [CANCELLED]-prefixed title → terminal-failure ring. + self.assertEqual(status_border_css('[CANCELLED] x'), TERMINAL_BOX_SHADOW) + + def test_failed_returns_terminal_box_shadow(self): + # D-08: [FAILED]-prefixed title → terminal-failure ring. + self.assertEqual(status_border_css('[FAILED] x'), TERMINAL_BOX_SHADOW) + + def test_weathered_returns_terminal_box_shadow(self): + # D-03/D-08 (Phase 23 Plan 02): [WEATHERED]-prefixed title → terminal-failure ring, + # same as [CANCELLED] -- both CampaignRun terminal run_status outcomes get the ring. + self.assertEqual(status_border_css('[WEATHERED] x'), TERMINAL_BOX_SHADOW) + + def test_unverified_returns_empty_string(self): + # D-09: placed bucket → '' (Phase 8's dashed border owns this distinction). + self.assertEqual(status_border_css('[UNVERIFIED] x'), '') + + def test_clean_title_returns_empty_string(self): + # D-09: no known prefix → '' (placed, no extra ring). + self.assertEqual(status_border_css('Some title'), '') + + def test_queued_box_shadow_differs_from_terminal(self): + # D-08: queued and terminal-failure are visually distinct. + self.assertNotEqual(QUEUED_BOX_SHADOW, TERMINAL_BOX_SHADOW) + + def test_no_dashed_in_queued_result(self): + # D-09: dashed border-style is reserved for Phase 8's is_verified cue. + self.assertNotIn('dashed', status_border_css('[QUEUED] x')) + + def test_no_dashed_in_terminal_result(self): + # D-09: terminal ring must not use dashed border-style. + self.assertNotIn('dashed', status_border_css('[EXPIRED] x')) + self.assertNotIn('dashed', status_border_css('[CANCELLED] x')) + self.assertNotIn('dashed', status_border_css('[FAILED] x')) + self.assertNotIn('dashed', status_border_css('[WEATHERED] x')) + + def test_no_dashed_in_placed_result(self): + # D-09: placed events return '' — inherently no dashed. + self.assertNotIn('dashed', status_border_css('[UNVERIFIED] x')) + self.assertNotIn('dashed', status_border_css('clean title')) + + +def _make_weeks(proposals): + """Build a minimal fake weeks structure from a flat list of proposal strings.""" + events = [SimpleNamespace(proposal=p) for p in proposals] + day = SimpleNamespace(all_day_events=events, events=[]) + return [[day]] + + +class VisibleProposalsTest(TestCase): + def test_groups_by_color_with_collision_handling(self): + # D-04: colliding proposal codes share one legend entry. + # Build expected mapping dynamically so the test is robust regardless + # of whether the chosen proposals actually collide. + proposals = ['PROP-A', 'PROP-B', 'PROP-C', ''] + weeks = _make_weeks(proposals) + + expected_by_color = {} + for p in proposals: + color = proposal_color(p) + normalized = (p or '').strip().upper() + label = normalized if normalized else CLASSICAL_SCHEDULE_LABEL + expected_by_color.setdefault(color, set()).add(label) + + result = visible_proposals(weeks) + self.assertEqual(len(result), len(expected_by_color)) + + for entry in result: + self.assertIn(entry['color'], expected_by_color) + actual_labels = set(entry['label'].split(', ')) + self.assertEqual(actual_labels, expected_by_color[entry['color']]) + + def test_neutral_slot_color_for_empty_proposal(self): + # D-05: empty-proposal event → NEUTRAL_SLOT_COLOR entry. + weeks = _make_weeks(['']) + result = visible_proposals(weeks) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]['color'], NEUTRAL_SLOT_COLOR) + + def test_neutral_slot_label_is_classical_schedule(self): + # D-06: empty-proposal legend entry is labeled 'Classical schedule'. + weeks = _make_weeks(['']) + result = visible_proposals(weeks) + self.assertEqual(result[0]['label'], CLASSICAL_SCHEDULE_LABEL) + + def test_neutral_slot_ordered_last(self): + # D-06 / 09-UI-SPEC Legend Layout: Classical schedule entry appears last. + weeks = _make_weeks(['PROP-A', '']) + result = visible_proposals(weeks) + self.assertGreater(len(result), 0) + self.assertEqual(result[-1]['color'], NEUTRAL_SLOT_COLOR) + self.assertEqual(result[-1]['label'], CLASSICAL_SCHEDULE_LABEL) + + def test_absent_proposal_not_in_result(self): + # D-02: only proposals present in weeks appear in the legend. + weeks = _make_weeks(['PROP-A']) + result = visible_proposals(weeks) + all_labels = ' '.join(e['label'] for e in result) + self.assertNotIn('PROP-B', all_labels) + + +class TextColorForBgTest(TestCase): + def test_all_palette_colors_return_white(self): + # DISPLAY-08: all 8 PROPOSAL_PALETTE entries achieve WCAG AA 4.5:1 with white text. + for hex_color in PROPOSAL_PALETTE: + with self.subTest(hex_color=hex_color): + self.assertEqual(text_color_for_bg(hex_color), '#fff') + + def test_neutral_slot_returns_white(self): + # DISPLAY-08: NEUTRAL_SLOT_COLOR (#5a6268) achieves WCAG AA with white text. + self.assertEqual(text_color_for_bg(NEUTRAL_SLOT_COLOR), '#fff') + + def test_bright_background_returns_black(self): + # DISPLAY-08: formula correctness — pure white background yields black text. + self.assertEqual(text_color_for_bg('#ffffff'), '#000') + + def test_pure_black_returns_white(self): + # DISPLAY-08: pure black background yields white text (maximum contrast). + self.assertEqual(text_color_for_bg('#000000'), '#fff') diff --git a/solsys_code/tests/test_calendar_template.py b/solsys_code/tests/test_calendar_template.py new file mode 100644 index 00000000..f068e1c1 --- /dev/null +++ b/solsys_code/tests/test_calendar_template.py @@ -0,0 +1,283 @@ +"""First view-level rendering test for tom_calendar's calendar.html override. + +Asserts the DISPLAY-02/03 dashed-border + tooltip markers appear for fallback-labeled +events only, on both the all-day and timed render branches, and that a CalendarEvent +with no CalendarEventTelescopeLabel sidecar row renders without raising (DISPLAY-01 +read-side default, A1). + +Phase 9 additions cover DISPLAY-04/05/06/07: proposal-color fills, [QUEUED] override +fix, status box-shadow rings, composition with Phase 8 dashed border, and the footer +legend with click-to-filter infrastructure. +""" + +from datetime import datetime +from datetime import timezone as dt_timezone + +from django.db import connection +from django.test import Client, TestCase +from django.test.utils import CaptureQueriesContext +from django.urls import reverse +from tom_calendar.models import CalendarEvent + +from solsys_code.models import CalendarEventTelescopeLabel +from solsys_code.templatetags.calendar_display_extras import proposal_color + +DASHED_BORDER_MARKER = '2px dashed rgba(0, 0, 0, 0.65)' +TOOLTIP_SUBSTRING = 'estimate' + +# Phase 9 marker constants (DISPLAY-05/06) — note: NO trailing semicolon so these work +# as substring matches against the CSS the tags emit (which does include the semicolon). +QUEUED_BOX_SHADOW = 'box-shadow: 0 0 0 2px rgba(0, 0, 0, 0.45)' +TERMINAL_BOX_SHADOW = 'box-shadow: 0 0 0 3px rgba(160, 0, 0, 0.55)' +# This is the old [QUEUED] background-color override that DISPLAY-05 requires removing. +# Note: assert the full `background-color:` prefix — the new queued box-shadow +# legitimately contains the bare rgba value as a substring (see plan Task 3 note). +OLD_QUEUED_GREY = 'background-color: rgba(0, 0, 0, 0.45)' +NEUTRAL_HEX = '#5a6268' + + +class CalendarTemplateTest(TestCase): + def setUp(self) -> None: + self.client = Client() + self.year = 2026 + self.month = 6 + + # All-day branch: start/end dates differ. + self.all_day_fallback = CalendarEvent.objects.create( + title='All-day fallback', + start_time=datetime(2026, 6, 10, 22, 0, tzinfo=dt_timezone.utc), + end_time=datetime(2026, 6, 11, 6, 0, tzinfo=dt_timezone.utc), + ) + CalendarEventTelescopeLabel.objects.create(event=self.all_day_fallback, is_verified=False) + + self.all_day_verified = CalendarEvent.objects.create( + title='All-day verified', + start_time=datetime(2026, 6, 12, 22, 0, tzinfo=dt_timezone.utc), + end_time=datetime(2026, 6, 13, 6, 0, tzinfo=dt_timezone.utc), + ) + CalendarEventTelescopeLabel.objects.create(event=self.all_day_verified, is_verified=True) + + self.all_day_no_row = CalendarEvent.objects.create( + title='All-day no sidecar row', + start_time=datetime(2026, 6, 14, 22, 0, tzinfo=dt_timezone.utc), + end_time=datetime(2026, 6, 15, 6, 0, tzinfo=dt_timezone.utc), + ) + + # Timed branch: start/end share the same date. + self.timed_fallback = CalendarEvent.objects.create( + title='Timed fallback', + start_time=datetime(2026, 6, 16, 22, 0, tzinfo=dt_timezone.utc), + end_time=datetime(2026, 6, 16, 23, 0, tzinfo=dt_timezone.utc), + ) + CalendarEventTelescopeLabel.objects.create(event=self.timed_fallback, is_verified=False) + + self.timed_verified = CalendarEvent.objects.create( + title='Timed verified', + start_time=datetime(2026, 6, 17, 22, 0, tzinfo=dt_timezone.utc), + end_time=datetime(2026, 6, 17, 23, 0, tzinfo=dt_timezone.utc), + ) + CalendarEventTelescopeLabel.objects.create(event=self.timed_verified, is_verified=True) + + self.timed_no_row = CalendarEvent.objects.create( + title='Timed no sidecar row', + start_time=datetime(2026, 6, 18, 22, 0, tzinfo=dt_timezone.utc), + end_time=datetime(2026, 6, 18, 23, 0, tzinfo=dt_timezone.utc), + ) + + # Phase 9 fixtures — proposal-color, status rings, composition (DISPLAY-04/05/06/07). + # All use June 2026 dates not already taken by Phase 8 fixtures above. + self.queued_event = CalendarEvent.objects.create( + title='[QUEUED] LTP2025A run', + proposal='LTP2025A-004', + start_time=datetime(2026, 6, 20, 22, 0, tzinfo=dt_timezone.utc), + end_time=datetime(2026, 6, 21, 6, 0, tzinfo=dt_timezone.utc), + ) + + self.terminal_event = CalendarEvent.objects.create( + title='[FAILED] LTP2025B run', + proposal='LTP2025B-012', + start_time=datetime(2026, 6, 22, 22, 0, tzinfo=dt_timezone.utc), + end_time=datetime(2026, 6, 23, 6, 0, tzinfo=dt_timezone.utc), + ) + + # Timed event with a proposal — exercises the timed proposal bullet (DISPLAY-04 both-branches). + self.timed_with_proposal = CalendarEvent.objects.create( + title='LTP2025A timed run', + proposal='LTP2025A-004', + start_time=datetime(2026, 6, 25, 10, 0, tzinfo=dt_timezone.utc), + end_time=datetime(2026, 6, 25, 11, 0, tzinfo=dt_timezone.utc), + ) + + # Empty-proposal all-day event — exercises the neutral slot (DISPLAY-04, DISPLAY-07). + self.no_proposal_event = CalendarEvent.objects.create( + title='Classical block', + proposal='', + start_time=datetime(2026, 6, 24, 22, 0, tzinfo=dt_timezone.utc), + end_time=datetime(2026, 6, 25, 6, 0, tzinfo=dt_timezone.utc), + ) + + # Pitfall 3 composition fixture: queued AND fallback-labeled timed event. + # Carries both the QUEUED box-shadow ring AND the Phase 8 dashed border. + # Contributes exactly 1 additional day-cell occurrence of DASHED_BORDER_MARKER. + self.queued_fallback_timed = CalendarEvent.objects.create( + title='[QUEUED] fallback run', + proposal='LTP2025A-004', + start_time=datetime(2026, 6, 27, 10, 0, tzinfo=dt_timezone.utc), + end_time=datetime(2026, 6, 27, 11, 0, tzinfo=dt_timezone.utc), + ) + CalendarEventTelescopeLabel.objects.create(event=self.queued_fallback_timed, is_verified=False) + + # The all-day fallback event spans 2 calendar days (Jun 10-11), so the calendar + # view's day-cell bucketing (offset_date(start) <= d <= offset_date(end)) renders + # it once per day cell it touches; the timed fallback event renders exactly once; + # queued_fallback_timed (Phase 9) is a timed fallback event contributing exactly 1. + self.num_fallback_day_cell_occurrences = 2 + 1 + 1 + + def _get_calendar(self): + return self.client.get(reverse('calendar:calendar'), {'year': self.year, 'month': self.month}) + + def test_calendar_renders_200_including_no_sidecar_row_events(self): + """Proves the silenced DoesNotExist path (A1): no-row events don't 500.""" + response = self._get_calendar() + self.assertEqual(response.status_code, 200) + + def test_fallback_events_get_dashed_border_and_tooltip(self): + response = self._get_calendar() + self.assertContains(response, DASHED_BORDER_MARKER) + self.assertContains(response, TOOLTIP_SUBSTRING) + + def test_dashed_border_count_matches_fallback_event_count_only(self): + """Verified and no-sidecar-row events (all-day and timed) must NOT get the dashed border. + + The all-day fallback event spans 2 day cells, so it contributes 2 occurrences of the + marker on its own; the timed fallback event contributes exactly 1; the Phase 9 + queued_fallback_timed event (is_verified=False) contributes 1 more. Verified and + no-sidecar-row events (both branches) must contribute 0. + """ + response = self._get_calendar() + content = response.content.decode() + self.assertEqual(content.count(DASHED_BORDER_MARKER), self.num_fallback_day_cell_occurrences) + + # --- Phase 9 tests: DISPLAY-04/05/06/07 --- + + def test_display05_old_queued_grey_background_color_is_gone(self): + """DISPLAY-05: the flat-grey [QUEUED] background-color override no longer appears. + + Asserts the full 'background-color: rgba(0, 0, 0, 0.45)' string is absent. + The new queued box-shadow legitimately contains the bare rgba value as a substring, + so only the background-color-prefixed form is checked here (plan Task 3 note, D-05). + """ + response = self._get_calendar() + content = response.content.decode() + self.assertNotIn(OLD_QUEUED_GREY, content) + + def test_display05_queued_event_renders_proposal_background_color(self): + """DISPLAY-05: [QUEUED] all-day event keeps its proposal-keyed background-color.""" + qhex = proposal_color('LTP2025A-004') + response = self._get_calendar() + content = response.content.decode() + self.assertIn(f'background-color: {qhex}', content) + + def test_display04_neutral_slot_color_present_for_empty_proposal_event(self): + """DISPLAY-04: empty-proposal event renders the neutral slot color (#5a6268).""" + response = self._get_calendar() + content = response.content.decode() + self.assertIn(NEUTRAL_HEX, content) + + def test_display04_timed_proposal_bullet_rendered(self): + """DISPLAY-04 (timed branch): timed event with proposal gets a proposal-color bullet.""" + qhex = proposal_color('LTP2025A-004') + response = self._get_calendar() + content = response.content.decode() + self.assertIn(f'color: {qhex}', content) + + def test_display06_queued_box_shadow_present(self): + """DISPLAY-06: [QUEUED] events carry the 2px queued ring.""" + response = self._get_calendar() + content = response.content.decode() + self.assertIn(QUEUED_BOX_SHADOW, content) + + def test_display06_terminal_box_shadow_present(self): + """DISPLAY-06: terminal-failure events carry the 3px red ring.""" + response = self._get_calendar() + content = response.content.decode() + self.assertIn(TERMINAL_BOX_SHADOW, content) + + def test_display06_queued_and_terminal_rings_are_visually_distinct(self): + """DISPLAY-06: the two status rings must be different strings (visual distinction).""" + self.assertNotEqual(QUEUED_BOX_SHADOW, TERMINAL_BOX_SHADOW) + + def test_display06_pitfall3_composition_dashed_and_queued_coexist(self): + """DISPLAY-06 + Pitfall 3: queued_fallback_timed carries BOTH the dashed border + (Phase 8 is_verified=False) AND the queued box-shadow ring (Phase 9 status).""" + response = self._get_calendar() + content = response.content.decode() + # Both signals coexist — Phase 8 signal not overwritten by Phase 9 status. + self.assertIn(DASHED_BORDER_MARKER, content) + self.assertIn(QUEUED_BOX_SHADOW, content) + # Exact count: 2 (all_day_fallback spans 2 days) + 1 (timed_fallback) + 1 (queued_fallback_timed) + self.assertEqual(content.count(DASHED_BORDER_MARKER), self.num_fallback_day_cell_occurrences) + + def test_display07_legend_swatch_markup_present(self): + """DISPLAY-07: the footer proposal legend contains .cal-legend-swatch elements.""" + response = self._get_calendar() + content = response.content.decode() + self.assertIn('cal-legend-swatch', content) + + def test_display07_classical_schedule_label_present_when_empty_proposal_events_visible(self): + """DISPLAY-07 D-06: the neutral-slot legend entry 'Classical schedule' appears + because no_proposal_event (proposal='') is visible this month.""" + response = self._get_calendar() + content = response.content.decode() + self.assertIn('Classical schedule', content) + + # --- Phase 12 tests: DISPLAY-08/09 --- + + def test_display08_inline_text_color_present_for_all_day_events(self): + """DISPLAY-08: all-day event divs carry an inline computed text color.""" + # DISPLAY-08: palette colors are dark, so computed text color is #fff. + response = self._get_calendar() + content = response.content.decode() + self.assertIn('color: #fff', content) + + def test_display08_important_color_rule_absent(self): + """DISPLAY-08: the hardcoded !important color override no longer appears in the page.""" + response = self._get_calendar() + content = response.content.decode() + self.assertNotIn('color: #fff !important', content) + + def test_display09_query_count_bounded(self): + """DISPLAY-09: query count does not grow when additional CalendarEvents are added.""" + # Baseline: count queries with setUp fixtures already present. + with CaptureQueriesContext(connection) as baseline_ctx: + self._get_calendar() + baseline_count = len(baseline_ctx) + + # Add one more CalendarEvent in the visible month and recount. + CalendarEvent.objects.create( + title='Extra event for N+1 test', + start_time=datetime(2026, 6, 28, 22, 0, tzinfo=dt_timezone.utc), + end_time=datetime(2026, 6, 29, 6, 0, tzinfo=dt_timezone.utc), + ) + with CaptureQueriesContext(connection) as extra_ctx: + self._get_calendar() + + # DISPLAY-09: query count must not grow with additional events. + self.assertEqual(len(extra_ctx), baseline_count) + + def test_display09_active_todo_count_renders_in_event_title(self): + """DISPLAY-09: active_todo_count annotation still shows todo parenthetical.""" + from tom_calendar.models import EventTodo + + # Create an event with an incomplete todo so the count parenthetical renders. + event_with_todo = CalendarEvent.objects.create( + title='Event with todo', + start_time=datetime(2026, 6, 28, 22, 0, tzinfo=dt_timezone.utc), + end_time=datetime(2026, 6, 29, 6, 0, tzinfo=dt_timezone.utc), + ) + EventTodo.objects.create(event=event_with_todo, description='Test task', is_completed=False) + + response = self._get_calendar() + content = response.content.decode() + # DISPLAY-09: the todo count parenthetical must appear in the rendered output. + self.assertIn('(1)', content) diff --git a/solsys_code/tests/test_calendar_utils.py b/solsys_code/tests/test_calendar_utils.py new file mode 100644 index 00000000..929c2977 --- /dev/null +++ b/solsys_code/tests/test_calendar_utils.py @@ -0,0 +1,176 @@ +from datetime import datetime, timedelta +from datetime import timezone as dt_timezone + +from django.test import TestCase +from tom_calendar.models import CalendarEvent + +from solsys_code.calendar_utils import insert_or_create_calendar_event + +# A fixed UTC sunset-like start time and a companion end time, used across the +# drift-tolerance tests below. +_START = datetime(2026, 7, 17, 22, 10, 56, tzinfo=dt_timezone.utc) +_END = datetime(2026, 7, 18, 11, 30, 0, tzinfo=dt_timezone.utc) +_TOLERANCE = timedelta(minutes=5) + + +class TestInsertOrCreateCalendarEventExactMatch(TestCase): + """Default (exact-equality) behaviour used by the URL-keyed sync commands.""" + + def test_url_lookup_creates_then_leaves_unchanged(self): + """A URL-keyed create-or-update creates once, then reports 'unchanged' on re-run.""" + lookup = {'url': 'https://example.test/obs/1'} + fields = {'title': 'Obs 1', 'start_time': _START, 'end_time': _END} + + event1, action1 = insert_or_create_calendar_event(lookup, fields) + event2, action2 = insert_or_create_calendar_event(lookup, fields) + + self.assertEqual(action1, 'created') + self.assertEqual(action2, 'unchanged') + self.assertEqual(event1.pk, event2.pk) + self.assertEqual(CalendarEvent.objects.count(), 1) + + def test_url_lookup_updates_on_changed_field(self): + """A changed field on a URL-keyed re-run reports 'updated' without duplicating.""" + lookup = {'url': 'https://example.test/obs/2'} + insert_or_create_calendar_event(lookup, {'title': 'Old', 'start_time': _START, 'end_time': _END}) + event, action = insert_or_create_calendar_event( + lookup, {'title': 'New', 'start_time': _START, 'end_time': _END} + ) + + self.assertEqual(action, 'updated') + self.assertEqual(event.title, 'New') + self.assertEqual(CalendarEvent.objects.count(), 1) + + def test_exact_start_time_key_duplicates_on_drift(self): + """Without a tolerance, a drifted start_time in the lookup key creates a duplicate. + + This documents the pre-fix failure mode: exact equality on a computed start_time + is fragile, which is exactly why load_telescope_runs opts into the tolerance below. + """ + key = {'telescope': 'Magellan-Baade', 'instrument': 'IMACS'} + insert_or_create_calendar_event({**key, 'start_time': _START}, {'title': 'A', 'end_time': _END}) + _event, action = insert_or_create_calendar_event( + {**key, 'start_time': _START + timedelta(seconds=2)}, {'title': 'A', 'end_time': _END} + ) + + self.assertEqual(action, 'created') + self.assertEqual(CalendarEvent.objects.count(), 2) + + +class TestInsertOrCreateCalendarEventStartTimeTolerance(TestCase): + """Proximity-matching behaviour used by load_telescope_runs (the bug fix).""" + + def _key(self) -> dict[str, str]: + return {'telescope': 'Magellan-Baade', 'instrument': 'IMACS'} + + def test_within_tolerance_no_field_change_is_unchanged(self): + """A re-ingest whose start_time drifted a few seconds, with no field change, is 'unchanged'.""" + event1, action1 = insert_or_create_calendar_event( + {**self._key(), 'start_time': _START}, + {'title': 'IMACS run', 'end_time': _END}, + start_time_tolerance=_TOLERANCE, + ) + event2, action2 = insert_or_create_calendar_event( + {**self._key(), 'start_time': _START + timedelta(seconds=2)}, + {'title': 'IMACS run', 'end_time': _END}, + start_time_tolerance=_TOLERANCE, + ) + + self.assertEqual(action1, 'created') + self.assertEqual(action2, 'unchanged') + self.assertEqual(event1.pk, event2.pk) + self.assertEqual(CalendarEvent.objects.count(), 1) + + def test_within_tolerance_keeps_original_start_time_pinned(self): + """A within-tolerance match must NOT rewrite the stored start_time (no churn).""" + insert_or_create_calendar_event( + {**self._key(), 'start_time': _START}, + {'title': 'IMACS run', 'end_time': _END}, + start_time_tolerance=_TOLERANCE, + ) + event, _action = insert_or_create_calendar_event( + {**self._key(), 'start_time': _START + timedelta(seconds=2)}, + {'title': 'IMACS run', 'end_time': _END}, + start_time_tolerance=_TOLERANCE, + ) + + # The stored start_time stays pinned to the first-ingested value. + self.assertEqual(event.start_time, _START) + + def test_within_tolerance_across_minute_boundary_still_matches(self): + """Drift that straddles a whole-minute boundary still matches (a window, not a bucket). + + 22:10:59 -> 22:11:01 would fall in different minute buckets, so any round/truncate + scheme would still duplicate; the +/- window centred on the target does not. + """ + near_minute = datetime(2026, 7, 17, 22, 10, 59, tzinfo=dt_timezone.utc) + insert_or_create_calendar_event( + {**self._key(), 'start_time': near_minute}, + {'title': 'IMACS run', 'end_time': _END}, + start_time_tolerance=_TOLERANCE, + ) + _event, action = insert_or_create_calendar_event( + {**self._key(), 'start_time': near_minute + timedelta(seconds=2)}, + {'title': 'IMACS run', 'end_time': _END}, + start_time_tolerance=_TOLERANCE, + ) + + self.assertEqual(action, 'unchanged') + self.assertEqual(CalendarEvent.objects.count(), 1) + + def test_within_tolerance_with_changed_field_updates_not_duplicates(self): + """A drifted re-ingest that also changed a real field is 'updated', never duplicated.""" + insert_or_create_calendar_event( + {**self._key(), 'start_time': _START}, + {'title': 'IMACS run', 'end_time': _END}, + start_time_tolerance=_TOLERANCE, + ) + event, action = insert_or_create_calendar_event( + {**self._key(), 'start_time': _START + timedelta(seconds=2)}, + {'title': 'IMACS run (proposed)', 'end_time': _END}, + start_time_tolerance=_TOLERANCE, + ) + + self.assertEqual(action, 'updated') + self.assertEqual(event.title, 'IMACS run (proposed)') + self.assertEqual(CalendarEvent.objects.count(), 1) + + def test_distinct_night_outside_tolerance_creates_new(self): + """A genuinely different night (~24h away) is outside the window and creates a new event. + + Confirms the tolerance can never merge two legitimately distinct nights for the + same telescope+instrument. + """ + insert_or_create_calendar_event( + {**self._key(), 'start_time': _START}, + {'title': 'IMACS run', 'end_time': _END}, + start_time_tolerance=_TOLERANCE, + ) + _event, action = insert_or_create_calendar_event( + {**self._key(), 'start_time': _START + timedelta(days=1)}, + {'title': 'IMACS run', 'end_time': _END + timedelta(days=1)}, + start_time_tolerance=_TOLERANCE, + ) + + self.assertEqual(action, 'created') + self.assertEqual(CalendarEvent.objects.count(), 2) + + def test_tolerance_scopes_match_by_other_lookup_keys(self): + """Proximity is scoped by the remaining lookup keys: a different instrument never matches. + + Two different instruments on the same telescope with near-identical start_times are + distinct events; the window must not merge them. + """ + insert_or_create_calendar_event( + {'telescope': 'Magellan-Baade', 'instrument': 'IMACS', 'start_time': _START}, + {'title': 'IMACS run', 'end_time': _END}, + start_time_tolerance=_TOLERANCE, + ) + _event, action = insert_or_create_calendar_event( + {'telescope': 'Magellan-Baade', 'instrument': 'LDSS3', 'start_time': _START + timedelta(seconds=2)}, + {'title': 'LDSS3 run', 'end_time': _END}, + start_time_tolerance=_TOLERANCE, + ) + + self.assertEqual(action, 'created') + self.assertEqual(CalendarEvent.objects.count(), 2) diff --git a/solsys_code/tests/test_campaign_approval.py b/solsys_code/tests/test_campaign_approval.py new file mode 100644 index 00000000..fd12117d --- /dev/null +++ b/solsys_code/tests/test_campaign_approval.py @@ -0,0 +1,2208 @@ +"""Tests for the staff-facing approval-queue write path (SUBMIT-03 / CAL-01/02/03 / D-01/D-02). + +Covers: staff-only gating on both the approval-queue GET and the decision-endpoint POST +(never a soft-filter -- a redirect, never 200-with-pending-content, per 16-RESEARCH.md Pitfall +7); the atomic conditional approve/reject transition and its proven double-approve no-op +(SUBMIT-03); the D-06 hybrid CAMPAIGN:{pk} CalendarEvent projection that fires only for a +single concrete night (window_start == window_end) with a resolved site -- a dip-corrected +sun_event() window for a ground site, a midnight-UTC placeholder for a space site +(CAL-01/CAL-02); no duplicate event and no ``modified`` churn on re-approve (CAL-03); and the +reject path (no event created). + +Uses ``TargetList.objects.create(...)`` for the campaign container and plain +``CampaignRun.objects.create(...)`` fixtures. This module never fixtures an individual +``tom_targets.models.Target`` at all (CampaignRun.target is left unset throughout), so +CLAUDE.md's non-sidereal-only target-factory convention doesn't even arise here. +""" + +from datetime import date, datetime, timezone +from html.parser import HTMLParser +from unittest.mock import MagicMock, patch + +import requests +from django.contrib.auth.models import User +from django.core.cache import cache +from django.template.loader import render_to_string +from django.test import TestCase, override_settings +from django.urls import reverse +from tom_calendar.models import CalendarEvent +from tom_targets.models import TargetList + +from solsys_code import campaign_utils +from solsys_code.campaign_tables import ApprovalQueueTable, CampaignRunTable +from solsys_code.campaign_utils import NEEDS_REVIEW_NAME_PREFIX, is_placeholder_observatory, resolve_site +from solsys_code.models import CampaignRun +from solsys_code.solsys_code_observatory.models import Observatory +from solsys_code.solsys_code_observatory.utils import MPCObscodeFetcher +from solsys_code.telescope_runs import sun_event + +CONTACT_PERSON = 'Jane Coordinator' +CONTACT_EMAIL = 'jane@example.org' + +# debug/site-search-degraded-pool-recurrence (bug #3): settings.CACHES is a FileBasedCache at +# tempfile.gettempdir() (/tmp), SHARED across processes -- and Django does NOT swap the cache +# backend for tests the way it swaps the database. Without this override, every cache.clear() +# in setUp/tearDown and every real build_site_candidates() call in the tests below reads, +# writes, and WIPES the same cache key ('mpc_obscode_candidates') the dev runserver serves +# site-search from. That is exactly the bug #3 regression: running this suite to verify a fix +# silently wiped the runserver's warmed ~5,700-entry MPC candidate pool, so the live +# site-search reverted to "No matches" until the next successful cold rebuild. Pinning the +# cache-touching test classes to an isolated in-memory LocMemCache keeps ALL test cache traffic +# off the shared file cache the runserver depends on. +ISOLATED_TEST_CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + 'LOCATION': 'campaign-tests-isolated', + } +} + +# Wave-0 fixture (SITE-01, RESEARCH.md Pattern 2/3): a small, representative slice of the +# real MPC bulk obscodes response shape ({obscode: {name_utf8, short_name, old_names, +# observations_type, longitude, ...}}). Deliberately does NOT include 'DCT' as a candidate +# string anywhere -- Pitfall 2 confirmed live that difflib cannot bridge the acronym/ +# nickname gap even against the full 5,636-string real pool, so 'DCT' must stay a genuine +# no-match case here too (G37's real MPC name is spelled out in full, never abbreviated). +BULK_MPC_FIXTURE = { + 'C65': { + 'name_utf8': 'Observatori Astronòmic del Montsec', + 'short_name': 'OAdM', + 'old_names': None, + 'observations_type': 'fixed', + 'longitude': 1.1937, + }, + '250': { + 'name_utf8': 'Hubble Space Telescope', + 'short_name': 'HST', + 'old_names': None, + 'observations_type': 'satellite', + 'longitude': None, + }, + 'G37': { + 'name_utf8': 'Lowell Discovery Telescope', + 'short_name': 'Lowell Discovery Telescope', + 'old_names': None, + 'observations_type': 'fixed', + 'longitude': -111.4223, + }, + 'W89': { + 'name_utf8': 'Siding Spring Observatory', + 'short_name': 'SSO', + 'old_names': None, + 'observations_type': 'fixed', + 'longitude': 149.0, + }, + 'F65': { + 'name_utf8': 'Faulkes Telescope South', + 'short_name': 'FTS', + 'old_names': None, + 'observations_type': 'fixed', + 'longitude': 149.0644, + }, + 'X09': { + 'name_utf8': 'Deep Random Survey, Rio Hurtado', + 'short_name': 'Deep Random Survey', + 'old_names': None, + 'observations_type': 'fixed', + 'longitude': -70.9, + }, +} + + +@override_settings(CACHES=ISOLATED_TEST_CACHES) +class CampaignApprovalTestBase(TestCase): + """Shared fixture: one campaign, one staff user, one non-staff user. + + Cache-isolated (bug #3, debug/site-search-degraded-pool-recurrence): the decide-endpoint + POST path resolves a site_selection via ``selection_to_obscode()`` -> + ``build_site_candidates()``, which reads/writes the ``mpc_obscode_candidates`` cache key. + The ``@override_settings(CACHES=ISOLATED_TEST_CACHES)`` here (inherited by every subclass) + keeps that traffic on an in-memory LocMemCache instead of the shared /tmp file cache the + dev runserver serves live site-search from. + """ + + @classmethod + def setUpTestData(cls) -> None: + cls.campaign = TargetList.objects.create(name='3I/ATLAS') + cls.staff_user = User.objects.create_user(username='staffcoordinator', password='pw', is_staff=True) + cls.non_staff_user = User.objects.create_user(username='regularobserver', password='pw', is_staff=False) + + def _make_pending_run(self, **overrides): + """Create a PENDING_REVIEW CampaignRun; kwargs override the default field set.""" + kwargs = { + 'campaign': self.campaign, + 'telescope_instrument': 'FTN/MuSCAT3', + 'site_raw': 'F65', + 'window_start': date(2026, 8, 1), + 'window_end': date(2026, 8, 1), + 'observation_details': 'Photometric monitoring', + 'contact_person': CONTACT_PERSON, + 'contact_email': CONTACT_EMAIL, + 'approval_status': CampaignRun.ApprovalStatus.PENDING_REVIEW, + } + kwargs.update(overrides) + return CampaignRun.objects.create(**kwargs) + + +class TestStaffGating(CampaignApprovalTestBase): + """T-16-03: anonymous/non-staff access must redirect, never render pending content.""" + + def test_anonymous_get_approval_queue_redirects(self): + run = self._make_pending_run() + response = self.client.get(reverse('campaigns:approval_queue')) + self.assertEqual(response.status_code, 302) + self.assertEqual(CampaignRun.objects.get(pk=run.pk).approval_status, CampaignRun.ApprovalStatus.PENDING_REVIEW) + + def test_non_staff_get_approval_queue_redirects(self): + self.client.login(username='regularobserver', password='pw') + response = self.client.get(reverse('campaigns:approval_queue')) + self.assertEqual(response.status_code, 302) + + def test_anonymous_post_decide_redirects_and_makes_no_change(self): + run = self._make_pending_run() + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.PENDING_REVIEW) + + def test_non_staff_post_decide_redirects_and_makes_no_change(self): + run = self._make_pending_run() + self.client.login(username='regularobserver', password='pw') + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.PENDING_REVIEW) + + def test_staff_get_approval_queue_succeeds(self): + self._make_pending_run() + self.client.login(username='staffcoordinator', password='pw') + response = self.client.get(reverse('campaigns:approval_queue')) + self.assertEqual(response.status_code, 200) + self.assertContains(response, 'Approval Queue') + + +class TestApproval(CampaignApprovalTestBase): + """SUBMIT-03: atomic approve/reject and the proven double-approve no-op.""" + + @classmethod + def setUpTestData(cls) -> None: + super().setUpTestData() + # D-06: a Tier-1-resolvable ground Observatory for the default fixture's site_raw + # ('F65') so approve's calendar projection (which now requires a resolved run.site) + # succeeds deterministically here, without a live MPC API call. + Observatory.objects.create( + obscode='F65', + name='Faulkes Telescope South', + short_name='FTS', + lat=-31.2727, + lon=149.0644, + altitude=1149.0, + timezone='Australia/Sydney', + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + + def setUp(self): + self.client.login(username='staffcoordinator', password='pw') + + def test_double_approve_is_noop(self): + run = self._make_pending_run() + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 1) + + # Second approve POST on the already-approved row must be a proven no-op. + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 1) + + def test_second_approve_surfaces_already_decided_warning(self): + run = self._make_pending_run() + self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}, follow=True + ) + messages = [str(m) for m in response.context['messages']] + self.assertIn('This run was already decided by someone else.', messages) + + def test_reject_path_sets_rejected_and_creates_no_event(self): + run = self._make_pending_run() + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'reject'}) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.REJECTED) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + + def test_invalid_action_returns_bad_request(self): + run = self._make_pending_run() + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'bogus'}) + self.assertEqual(response.status_code, 400) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.PENDING_REVIEW) + + def test_approving_already_resolved_site_does_not_call_resolve_site(self): + """SITE-03/D-06: a pre-set run.site is trusted and never re-resolved on approve.""" + observatory = Observatory.objects.get(obscode='F65') + run = self._make_pending_run(site=observatory, site_needs_review=False) + with patch('solsys_code.campaign_views.resolve_site') as mock_resolve_site: + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + self.assertEqual(response.status_code, 302) + mock_resolve_site.assert_not_called() + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(run.site_id, observatory.pk) + + def test_projection_failure_reverts_site_stays_set_second_approve_skips_resolve_site(self): + """RESEARCH.md Pitfall 3 regression: a projection failure reverts approval_status to + PENDING_REVIEW while leaving run.site set (D-06's clobber-fix guard); a second approve + POST must not re-call resolve_site() (the pre-fix bug re-ran the MPC fetch here).""" + run = self._make_pending_run() + with patch( + 'solsys_code.campaign_views.insert_or_create_calendar_event', + side_effect=RuntimeError('boom'), + ): + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.PENDING_REVIEW) + self.assertIsNotNone(run.site) + resolved_site_pk = run.site.pk + + with patch('solsys_code.campaign_views.resolve_site') as mock_resolve_site: + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + self.assertEqual(response.status_code, 302) + mock_resolve_site.assert_not_called() + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(run.site.pk, resolved_site_pk) + + def test_oversized_site_selection_is_flagged_with_no_network_call_or_fabrication(self): + """T-21-04: an oversized site_selection is flagged by resolve_site's existing + _MAX_OBSCODE_LEN guard -- no tier attempted, no network call, no fabricated Observatory.""" + run = self._make_pending_run() + oversized = 'X' * (Observatory._meta.get_field('obscode').max_length + 1) + with patch('solsys_code.campaign_utils.MPCObscodeFetcher.query') as mock_query: + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'approve', 'site_selection': oversized}, + ) + self.assertEqual(response.status_code, 302) + mock_query.assert_not_called() + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertIsNone(run.site) + self.assertTrue(run.site_needs_review) + self.assertEqual(Observatory.objects.filter(obscode=oversized).count(), 0) + + +class TestCalendarProjection(CampaignApprovalTestBase): + """D-06/CAL-01/CAL-02: approving a single-night run with a resolved site projects a + CAMPAIGN:{pk} event -- a dip-corrected sun_event() window for a ground site, a + midnight-UTC placeholder for a space site. A range, TBD run, missing + telescope_instrument, or a sun_event() ValueError all project nothing (the last of + these without reverting the already-committed approval). + """ + + @classmethod + def setUpTestData(cls) -> None: + super().setUpTestData() + # Tier-1-resolvable so approve's site resolution never needs a live MPC API call. + cls.ground_site = Observatory.objects.create( + obscode='F65', + name='Faulkes Telescope South', + short_name='FTS', + lat=-31.2727, + lon=149.0644, + altitude=1149.0, + timezone='Australia/Sydney', + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + + def setUp(self): + self.client.login(username='staffcoordinator', password='pw') + + def test_approve_single_night_ground_run_creates_dip_corrected_calendar_event(self): + run = self._make_pending_run() + self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + event = CalendarEvent.objects.get(url=f'CAMPAIGN:{run.pk}') + expected_sunset, expected_sunrise = sun_event(self.ground_site, run.window_start, kind='sun') + self.assertEqual(event.start_time, expected_sunset.to_datetime(timezone=timezone.utc).replace(microsecond=0)) + self.assertEqual(event.end_time, expected_sunrise.to_datetime(timezone=timezone.utc).replace(microsecond=0)) + self.assertEqual(event.target_list_id, self.campaign.pk) + self.assertEqual(event.telescope, run.telescope_instrument) + + def test_approve_single_night_space_run_creates_midnight_utc_placeholder_event(self): + space_site = Observatory.objects.create( + obscode='250', + name='Test Space Telescope', + short_name='TST', + observations_type=Observatory.SATELLITE_OBSTYPE, + ) + run = self._make_pending_run(site_raw=space_site.obscode) + self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + event = CalendarEvent.objects.get(url=f'CAMPAIGN:{run.pk}') + self.assertEqual(event.start_time, datetime(2026, 8, 1, 0, 0, tzinfo=timezone.utc)) + self.assertEqual(event.end_time, datetime(2026, 8, 1, 23, 59, tzinfo=timezone.utc)) + + def test_approve_range_run_creates_no_calendar_event(self): + run = self._make_pending_run(window_start=date(2026, 8, 1), window_end=date(2026, 8, 15)) + self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + + def test_approve_tbd_run_creates_no_calendar_event(self): + run = self._make_pending_run(window_start=None, window_end=None) + self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + + def test_approve_without_telescope_instrument_creates_no_calendar_event(self): + run = self._make_pending_run(telescope_instrument='') + self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(CalendarEvent.objects.count(), 0) + + def test_sun_event_valueerror_skips_projection_without_reverting_approval(self): + """Pitfall 7: a sun_event() ValueError (e.g. blank site.timezone) must be logged and + skipped, never reach the broad except Exception that reverts a half-committed + approval back to PENDING_REVIEW.""" + run = self._make_pending_run() + with patch('solsys_code.campaign_views.sun_event', side_effect=ValueError('no crossings')): + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + + +class TestRunStatusChange(CampaignApprovalTestBase): + """D-03/D-04/D-05: staff mark an APPROVED run cancelled or weathered from the Decided + table, and the linked CAMPAIGN:{pk} CalendarEvent (if one exists) updates in place with + a distinct terminal title prefix. A range/TBD/unresolved-site run that never had a + projected event is handled without crashing or fabricating one (RESEARCH Pitfall 1). A + non-APPROVED run, and a lost-update race between the guard read and the conditional + write (REVIEW finding #1), are both rejected/short-circuited server-side without a 500 + or a calendar mutation. + """ + + @classmethod + def setUpTestData(cls) -> None: + super().setUpTestData() + # Tier-1-resolvable ground Observatory so a single-night run's approval projects a + # CAMPAIGN:{pk} event deterministically, without a live MPC API call (mirrors + # TestCalendarProjection's ground_site fixture). + cls.ground_site = Observatory.objects.create( + obscode='F65', + name='Faulkes Telescope South', + short_name='FTS', + lat=-31.2727, + lon=149.0644, + altitude=1149.0, + timezone='Australia/Sydney', + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + + def setUp(self): + self.client.login(username='staffcoordinator', password='pw') + + def _make_approved_single_night_run(self, **overrides): + """Create+approve a single-night, resolved-site run so a CAMPAIGN:{pk} event exists.""" + run = self._make_pending_run(**overrides) + self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + run.refresh_from_db() + return run + + def test_mark_cancelled_single_night_updates_existing_event_in_place(self): + run = self._make_approved_single_night_run() + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 1) + + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'mark_cancelled'}) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.run_status, CampaignRun.RunStatus.CANCELLED) + events = CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}') + self.assertEqual(events.count(), 1) + event = events.get() + self.assertTrue(event.title.startswith('[CANCELLED] ')) + # REVIEW finding #3: the description reflects the status change, not byte-identical + # to the original projection description. + self.assertIn('Run status: Cancelled', event.description) + + def test_mark_weather_failure_uses_distinct_weathered_prefix(self): + run = self._make_approved_single_night_run() + + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'mark_weather_failure'} + ) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.run_status, CampaignRun.RunStatus.WEATHER_TECH_FAILURE) + event = CalendarEvent.objects.get(url=f'CAMPAIGN:{run.pk}') + self.assertTrue(event.title.startswith('[WEATHERED] ')) + self.assertFalse(event.title.startswith('[CANCELLED]')) + self.assertIn('Run status: Weather/Technical Failure', event.description) + + def test_mark_range_window_run_does_not_crash_and_creates_no_event(self): + run = self._make_approved_single_night_run(window_start=date(2026, 8, 1), window_end=date(2026, 8, 15)) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'mark_cancelled'}) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.run_status, CampaignRun.RunStatus.CANCELLED) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + + def test_mark_status_on_non_approved_run_rejected(self): + run = self._make_pending_run() # still PENDING_REVIEW -- never approved + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'mark_cancelled'}, follow=True + ) + self.assertEqual(response.status_code, 200) + run.refresh_from_db() + self.assertEqual(run.run_status, CampaignRun.RunStatus.REQUESTED) + messages_list = [str(m) for m in response.context['messages']] + self.assertIn('This run has not been approved yet.', messages_list) + + def test_mark_status_lost_update_race_warns_no_calendar_mutation(self): + """REVIEW finding #1 backstop: the guard read sees APPROVED (a stale in-memory + object), but the DB row's approval_status has actually changed to PENDING_REVIEW by + the time the conditional `.update()` runs -- mirrors _resolve_site()'s + `claimed == 0` guard. Must not raise CampaignRun.DoesNotExist / 500, must not + mutate the DB run_status, and must not touch any CalendarEvent. + """ + run = self._make_approved_single_night_run() + event_count_before = CalendarEvent.objects.count() + stale_run = CampaignRun.objects.get(pk=run.pk) + # Simulate the race after the stale read: the row is no longer APPROVED. + CampaignRun.objects.filter(pk=run.pk).update(approval_status=CampaignRun.ApprovalStatus.PENDING_REVIEW) + + with patch('solsys_code.campaign_views.get_object_or_404', return_value=stale_run): + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'mark_cancelled'}, follow=True + ) + self.assertEqual(response.status_code, 200) + messages_list = [str(m) for m in response.context['messages']] + self.assertTrue(any('could not be updated' in m for m in messages_list)) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.PENDING_REVIEW) + self.assertEqual(run.run_status, CampaignRun.RunStatus.REQUESTED) + self.assertEqual(CalendarEvent.objects.count(), event_count_before) + + def test_unknown_action_still_bad_request(self): + run = self._make_approved_single_night_run() + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'mark_bogus'}) + self.assertEqual(response.status_code, 400) + run.refresh_from_db() + self.assertEqual(run.run_status, CampaignRun.RunStatus.REQUESTED) + + def test_mark_status_anonymous_or_non_staff_makes_no_change(self): + run = self._make_approved_single_night_run() + self.client.logout() + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'mark_cancelled'}) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.run_status, CampaignRun.RunStatus.REQUESTED) + + self.client.login(username='regularobserver', password='pw') + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'mark_cancelled'}) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.run_status, CampaignRun.RunStatus.REQUESTED) + + +class TestDecidedTableStatusActions(CampaignApprovalTestBase): + """D-04 (Plan 02): the Decided table's Mark Cancelled/Mark Weathered action is gated by + the independent ``status_actions`` flag, never by flipping ``show_actions`` -- the Site + column's plain-text fallback (RESEARCH Pitfall 3) must stay completely untouched. + """ + + def setUp(self): + self.client.login(username='staffcoordinator', password='pw') + + def test_decided_table_renders_status_actions_for_approved_run(self): + self._make_pending_run(approval_status=CampaignRun.ApprovalStatus.APPROVED) + + response = self.client.get(reverse('campaigns:approval_queue')) + + content = response.content.decode() + self.assertIn('name="action" value="mark_cancelled"', content) + self.assertIn('name="action" value="mark_weather_failure"', content) + self.assertIn('Mark Cancelled', content) + self.assertIn('Mark Weathered', content) + + def test_decided_table_no_status_actions_for_rejected_run(self): + self._make_pending_run(approval_status=CampaignRun.ApprovalStatus.REJECTED) + + response = self.client.get(reverse('campaigns:approval_queue')) + + content = response.content.decode() + self.assertNotIn('name="action" value="mark_cancelled"', content) + self.assertNotIn('name="action" value="mark_weather_failure"', content) + + def test_decided_table_site_column_stays_plain_text(self): + # REJECTED (not APPROVED) so this row can never also land in review_table -- keeps + # the assertion scoped purely to the Decided table's own render_site() output. + run = self._make_pending_run( + approval_status=CampaignRun.ApprovalStatus.REJECTED, site=None, site_raw='DCT', site_needs_review=False + ) + + response = self.client.get(reverse('campaigns:approval_queue')) + + content = response.content.decode() + self.assertIn('DCT', content) + self.assertNotIn(f'id="site-input-{run.pk}"', content) + self.assertNotIn('name="site_selection"', content) + + +class TestApprovalQueueColumns(TestCase): + """UAT Test 14 gap closure (16-05): ApprovalQueueTable is trimmed/reordered for triage, + CampaignRunTable stays spreadsheet-parity (Phase 15 D-09 regression guard). + + No DB rows are needed -- both tables are built with an empty data list; only the + declared column contract (``.columns``) is under test here. + """ + + def test_actions_leads_approval_queue_table(self): + column_names = [column.name for column in ApprovalQueueTable([]).columns] + self.assertEqual(column_names[0], 'actions') + + def test_approval_queue_table_excludes_post_observation_columns(self): + column_names = {column.name for column in ApprovalQueueTable([]).columns} + self.assertEqual(column_names & {'weather', 'observation_outcome', 'publication_plans'}, set()) + + def test_campaign_run_table_unchanged_by_approval_queue_trim(self): + """D-09 regression guard: the fix is scoped to ApprovalQueueTable only.""" + column_names = {column.name for column in CampaignRunTable([]).columns} + self.assertTrue({'weather', 'observation_outcome', 'publication_plans'} <= column_names) + self.assertNotIn('actions', column_names) + + +class TestApprovalQueueSiteVisibility(CampaignApprovalTestBase): + """Regression coverage for the visibility gap: pending runs (site_needs_review=False, + per D-07) must still surface their submitted site_raw text in the site column, not just + runs where resolution ran and failed.""" + + def test_pending_unresolved_site_shows_site_raw(self): + run = self._make_pending_run(site=None, site_raw='DCT', site_needs_review=False) + cell = CampaignRunTable([run]).rows[0].get_cell('site') + self.assertIn('DCT', cell) + + def test_pending_blank_site_raw_renders_empty_cell(self): + run = self._make_pending_run(site=None, site_raw='', site_needs_review=False) + cell = CampaignRunTable([run]).rows[0].get_cell('site') + self.assertEqual(cell, '') + + def test_resolution_failed_site_still_shows_site_raw_with_failure_indicator(self): + run = self._make_pending_run(site=None, site_raw='DCT', site_needs_review=True) + cell = CampaignRunTable([run]).rows[0].get_cell('site') + self.assertIn('DCT', cell) + self.assertIn('exclamation-triangle', cell) + + +class TestApprovalSiteResolution(CampaignApprovalTestBase): + """Approving an unresolvable free-text site must not fabricate a placeholder + Observatory row (unlike the already-vetted CSV import path), and must not block + approval (D-07).""" + + def setUp(self): + self.client.login(username='staffcoordinator', password='pw') + # Keep tier 2 deterministic and offline: simulate an MPC miss/no-network so + # resolution always falls through past tier 2 to the create_placeholder branch. + patcher = patch( + 'solsys_code.campaign_utils.MPCObscodeFetcher.query', + side_effect=requests.exceptions.RequestException, + ) + patcher.start() + self.addCleanup(patcher.stop) + + def test_approving_unresolvable_free_text_site_creates_no_placeholder_observatory(self): + run = self._make_pending_run(site_raw='DCT') + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertIsNone(run.site) + self.assertTrue(run.site_needs_review) + self.assertEqual(Observatory.objects.count(), 0) + + def test_resolve_site_create_placeholder_false_creates_no_observatory(self): + observatory, needs_review = resolve_site('DCT', create_placeholder=False) + self.assertIsNone(observatory) + self.assertTrue(needs_review) + self.assertEqual(Observatory.objects.count(), 0) + + def test_resolve_site_default_still_creates_placeholder_observatory(self): + """CSV-import path (default create_placeholder=True) is unaffected.""" + observatory, needs_review = resolve_site('DCT') + self.assertIsNotNone(observatory) + self.assertEqual(observatory.obscode, 'DCT') + self.assertTrue(needs_review) + self.assertEqual(Observatory.objects.count(), 1) + + def test_resolve_site_tier1_hit_on_existing_placeholder_still_flags_review(self): + """CR-01 (22-REVIEW.md re-review): a Tier 1 hit against a *pre-existing* tier-3 + placeholder (e.g. a repeat CSV Site Code whose first row already created it) must + still report needs_review=True -- it is not a genuine resolution just because an + Observatory row exists for that obscode.""" + placeholder = Observatory.objects.create(obscode='DCT', name=f'{NEEDS_REVIEW_NAME_PREFIX}DCT', short_name='DCT') + + site, needs_review = resolve_site('DCT', create_placeholder=False) + + self.assertEqual(site, placeholder) + self.assertTrue(needs_review) + self.assertEqual(Observatory.objects.count(), 1) # no second placeholder fabricated + + +class TestSiteSelectionResolution(CampaignApprovalTestBase): + """SITE-02: the staff-submitted site_selection value drives approve-time resolution.""" + + def setUp(self): + self.client.login(username='staffcoordinator', password='pw') + + def test_staff_typed_existing_obscode_resolves_via_site_selection_tier_1_hit(self): + """A tier-1 hit (existing Observatory) resolves with no fabrication.""" + Observatory.objects.create( + obscode='G37', + name='Lowell Discovery Telescope', + short_name='LDT', + lat=34.744, + lon=-111.4223, + altitude=2361.0, + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + run = self._make_pending_run(site_raw='Lowell Discvery Tel') # typo -- never resolves via site_raw + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'approve', 'site_selection': 'G37'}, + ) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(run.site.obscode, 'G37') + self.assertFalse(run.site_needs_review) + self.assertEqual(Observatory.objects.count(), 1) + + def test_unresolvable_site_selection_leaves_observatory_count_unchanged(self): + """Regression on 260705-l1v's invariant: an unresolvable site_selection on approve + creates no placeholder Observatory.""" + run = self._make_pending_run() + with patch( + 'solsys_code.campaign_utils.MPCObscodeFetcher.query', + side_effect=requests.exceptions.RequestException, + ): + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'approve', 'site_selection': 'NOWHERE'}, + ) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertIsNone(run.site) + self.assertTrue(run.site_needs_review) + self.assertEqual(Observatory.objects.count(), 0) + + def test_approve_re_resolves_when_existing_site_is_a_placeholder(self): + """WR-01 (22-REVIEW.md re-review): a PENDING_REVIEW run whose ``site`` already + points at a tier-3 placeholder Observatory (not None) must still re-enter site + resolution on approve, mirroring ``_resolve_site()``'s placeholder-aware guard -- + not just the site-is-None case. Without the fix, ``run.site is None`` is False here + (a placeholder Observatory is still an Observatory), so resolution never runs and + the run stays pointed at the unusable placeholder.""" + Observatory.objects.create( + obscode='G37', + name='Lowell Discovery Telescope', + short_name='LDT', + lat=34.744, + lon=-111.4223, + altitude=2361.0, + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + placeholder = Observatory.objects.create(obscode='DCT', name=f'{NEEDS_REVIEW_NAME_PREFIX}DCT', short_name='DCT') + run = self._make_pending_run(site=placeholder, site_raw='DCT', site_needs_review=True) + + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'approve', 'site_selection': 'G37'}, + ) + + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(run.site.obscode, 'G37') + self.assertFalse(run.site_needs_review) + + +class TestSiteSelectionNameCandidateResolution(CampaignApprovalTestBase): + """Permanent CR-01 regression (21-REVIEW-FIX.md / 21-VERIFICATION.md): a name/ + short_name/old_names display-string ``site_selection`` candidate -- NOT a literal + obscode -- submitted through the real ``campaigns:decide`` POST resolves ``run.site`` + via ``CampaignRunDecisionView.post()``'s ``selection_to_obscode()`` obscode mapping + (``campaign_utils``). ``TestSiteSelectionResolution`` above only exercises the + literal-obscode case (``'G37'`` passes through the mapping unchanged), so it never + proves the display-string lookup itself works. The verifier independently confirmed + this behavior with a temporary end-to-end test (written, run, then removed per + verifier convention) before this class existed; this class makes that coverage + permanent. + """ + + def setUp(self): + cache.clear() + self.client.login(username='staffcoordinator', password='pw') + self.observatory = Observatory.objects.create( + obscode='G37', + name='Lowell Discovery Telescope', + short_name='LDT', + lat=34.744, + lon=-111.4223, + altitude=2361.0, + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + # Explicit candidate pool mapping a name (name_utf8, from the fixture), a + # short_name, AND an old_names string all to obscode 'G37' -- the three + # display-string candidate types build_site_candidates()/_flatten_mpc_candidates() + # produce (RESEARCH.md Open Question 2). + candidate_pool = { + **campaign_utils._flatten_mpc_candidates(BULK_MPC_FIXTURE), + 'LDT': 'G37', + 'Historic Lowell Reflector': 'G37', + } + # The selection->obscode mapping lives in campaign_utils.selection_to_obscode(), which + # calls campaign_utils.build_site_candidates() -- patch it at that layer (not the view + # import site) so the decide POST's mapping sees this deterministic pool. + patcher = patch('solsys_code.campaign_utils.build_site_candidates', return_value=candidate_pool) + patcher.start() + self.addCleanup(patcher.stop) + + def tearDown(self): + cache.clear() + + def test_name_short_name_and_old_names_candidates_resolve_via_real_decide_post(self): + candidates = { + 'name_utf8': 'Lowell Discovery Telescope', + 'short_name': 'LDT', + 'old_names': 'Historic Lowell Reflector', + } + for index, (candidate_type, site_selection) in enumerate(candidates.items()): + with self.subTest(candidate_type=candidate_type, site_selection=site_selection): + # Distinct window_start per iteration -- CampaignRun's natural-key + # UniqueConstraint keys on (campaign, telescope_instrument, window_start, + # window_end), so three runs sharing _make_pending_run()'s default window + # would otherwise collide on the second/third create(). + run_date = date(2026, 8, 1 + index) + run = self._make_pending_run( + site_raw='Lowell Discvery Tel', # typo -- never self-resolves + window_start=run_date, + window_end=run_date, + ) + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'approve', 'site_selection': site_selection}, + ) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(run.site.obscode, 'G37') + self.assertEqual(run.site_id, self.observatory.pk) + self.assertFalse(run.site_needs_review) + self.assertEqual(Observatory.objects.count(), 1) + + +class TestSitesNeedingReview(CampaignApprovalTestBase): + """D-06/D-07/D-08/22-REVIEWS.md findings 3/5/6/8c: the resolve_site decision action for + approved runs whose site never resolved (``site_needs_review=True``). + + Fixture convention mirrors TestApproval/TestCalendarProjection: a Tier-1-resolvable + ground Observatory ('F65') so resolution never needs a live MPC API call. + """ + + @classmethod + def setUpTestData(cls) -> None: + super().setUpTestData() + cls.ground_site = Observatory.objects.create( + obscode='F65', + name='Faulkes Telescope South', + short_name='FTS', + lat=-31.2727, + lon=149.0644, + altitude=1149.0, + timezone='Australia/Sydney', + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + + def setUp(self): + self.client.login(username='staffcoordinator', password='pw') + + def _make_needs_review_run(self, **overrides): + """An APPROVED run with site_needs_review=True (the dead end this phase closes).""" + kwargs = { + 'approval_status': CampaignRun.ApprovalStatus.APPROVED, + 'site': None, + 'site_needs_review': True, + } + kwargs.update(overrides) + return self._make_pending_run(**kwargs) + + def test_resolve_success_single_night_ground_run_projects_calendar_event(self): + run = self._make_needs_review_run(site_raw='F65') + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'F65'}, + follow=True, + ) + self.assertEqual(response.status_code, 200) + run.refresh_from_db() + self.assertEqual(run.site_id, self.ground_site.pk) + self.assertFalse(run.site_needs_review) + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 1) + messages_list = [str(m) for m in response.context['messages']] + self.assertIn('Site resolved — run added to the calendar.', messages_list) + + def test_resolve_never_re_resolves_already_set_site_but_retries_projection(self): + """D-06/finding 8c: a run with site already set (the projection-failed retry state) + must never re-call resolve_site, but its projection IS re-attempted and the flag + clears on success.""" + run = self._make_needs_review_run(site=self.ground_site, site_raw='F65') + with patch('solsys_code.campaign_views.resolve_site') as mock_resolve_site: + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site'}, + ) + self.assertEqual(response.status_code, 302) + mock_resolve_site.assert_not_called() + run.refresh_from_db() + self.assertEqual(run.site_id, self.ground_site.pk) + self.assertFalse(run.site_needs_review) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 1) + + def test_resolve_retryable_projection_failure_stays_approved_site_saved_flag_stays_true(self): + """Finding 3: a projection failure must not revert approval, must keep the resolved + site, and must keep site_needs_review=True so the row stays in the retry surface.""" + run = self._make_needs_review_run(site_raw='F65') + with patch('solsys_code.campaign_views._project_calendar_event', side_effect=RuntimeError('boom')): + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'F65'}, + follow=True, + ) + self.assertEqual(response.status_code, 200) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(run.site_id, self.ground_site.pk) + self.assertTrue(run.site_needs_review) + messages_list = [str(m) for m in response.context['messages']] + self.assertTrue(any('calendar entry' in m for m in messages_list)) + + # Finding 3: the retry surface is preserved -- a subsequent staff GET of the + # approval queue still lists the run in review_table's underlying data (not just + # the model field). + queue_response = self.client.get(reverse('campaigns:approval_queue')) + review_table = queue_response.context['review_table'] + self.assertIn(run.pk, [row.record.pk for row in review_table.rows]) + + def test_resolve_blank_timezone_site_keeps_review_flag_and_creates_no_event(self): + """CR-01 (22-REVIEW.md): resolving to a site whose ``timezone`` is blank -- exactly + what ``MPCObscodeFetcher.to_observatory()`` (Tier 2) produces, since it never sets + ``timezone`` -- must not silently report success. ``sun_event()`` raises ``ValueError`` + for a blank timezone, and ``_project_calendar_event()`` must now re-raise it (rather + than swallowing it into a bare ``False``) so ``_resolve_site()``'s existing + non-reverting except block treats this the same as any other projection failure: + keep ``site_needs_review=True``, warn instead of claiming success, and create no + ``CalendarEvent``. Fixtured directly as a local Observatory (Tier 1 hit) with a blank + ``timezone`` rather than mocking the MPC fetch -- CR-01 only cares about the blank + timezone, not which tier produced it.""" + blank_tz_site = Observatory.objects.create( + obscode='T99', + name='Blank Timezone Site', + short_name='BTS', + lat=-30.0, + lon=149.0, + altitude=1000.0, + timezone='', + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + run = self._make_needs_review_run(site_raw='T99') + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'T99'}, + follow=True, + ) + self.assertEqual(response.status_code, 200) + run.refresh_from_db() + self.assertEqual(run.site_id, blank_tz_site.pk) + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertTrue(run.site_needs_review) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + messages_list = [str(m) for m in response.context['messages']] + self.assertNotIn('Site resolved.', messages_list) + self.assertTrue(any('calendar entry' in m for m in messages_list)) + + # Finding 3 (still holds under CR-01): the retry surface is preserved -- the run + # stays listed in review_table's underlying data. + queue_response = self.client.get(reverse('campaigns:approval_queue')) + review_table = queue_response.context['review_table'] + self.assertIn(run.pk, [row.record.pk for row in review_table.rows]) + + def test_resolve_lost_race_no_op_warns(self): + """Finding 5: a concurrent resolution landing between the fresh fetch and the site + write must make the loser's claim update match 0 rows -- no write, no projection.""" + run = self._make_needs_review_run(site_raw='F65') + + def _racing_resolve_site(obscode_selection, create_placeholder=False): + # Simulate the second staff member's POST winning the race: directly resolve + # the row's site in the DB before this (the loser's) call returns. + CampaignRun.objects.filter(pk=run.pk).update(site=self.ground_site, site_needs_review=False) + return self.ground_site, False + + with patch('solsys_code.campaign_views.resolve_site', side_effect=_racing_resolve_site): + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'F65'}, + follow=True, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + messages_list = [str(m) for m in response.context['messages']] + self.assertIn("This run's site was already resolved by someone else.", messages_list) + + def test_resolve_rejects_pending_review_run(self): + run = self._make_pending_run() + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'F65'}, + follow=True, + ) + self.assertEqual(response.status_code, 200) + run.refresh_from_db() + self.assertIsNone(run.site) + messages_list = [str(m) for m in response.context['messages']] + self.assertIn('This run is not awaiting site resolution.', messages_list) + + def test_resolve_rejects_already_resolved_run(self): + run = self._make_needs_review_run(site=self.ground_site, site_needs_review=False) + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'F65'}, + follow=True, + ) + self.assertEqual(response.status_code, 200) + messages_list = [str(m) for m in response.context['messages']] + self.assertIn('This run is not awaiting site resolution.', messages_list) + + def test_resolve_range_tbd_run_clears_flag_with_no_calendar_event(self): + run = self._make_needs_review_run(site_raw='F65', window_start=date(2026, 8, 1), window_end=date(2026, 8, 15)) + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'F65'}, + follow=True, + ) + self.assertEqual(response.status_code, 200) + run.refresh_from_db() + self.assertEqual(run.site_id, self.ground_site.pk) + self.assertFalse(run.site_needs_review) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + messages_list = [str(m) for m in response.context['messages']] + self.assertIn('Site resolved.', messages_list) + + def test_resolve_unresolvable_selection_leaves_site_none_and_flag_true(self): + run = self._make_needs_review_run(site_raw='') + with patch( + 'solsys_code.campaign_utils.MPCObscodeFetcher.query', + side_effect=requests.exceptions.RequestException, + ): + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'NOWHERE'}, + follow=True, + ) + self.assertEqual(response.status_code, 200) + run.refresh_from_db() + self.assertIsNone(run.site) + self.assertTrue(run.site_needs_review) + messages_list = [str(m) for m in response.context['messages']] + self.assertIn( + 'Could not resolve that site. Try a different search term or an exact MPC code, ' + 'or use Create new Observatory.', + messages_list, + ) + + def test_review_table_context_lists_only_approved_needs_review_runs(self): + """D-07: review_table lists APPROVED+site_needs_review=True runs only -- not + pending, not resolved-approved -- INCLUDING a projection-failed retry row (site set, + flag still True), since the filter is on the flag alone.""" + self._make_pending_run(site_raw='F65') # PENDING_REVIEW -- must not appear + self._make_needs_review_run(site_raw='F65', window_start=date(2026, 8, 2), window_end=date(2026, 8, 2)) + resolved_approved = self._make_needs_review_run( + site=self.ground_site, + site_needs_review=False, + window_start=date(2026, 8, 3), + window_end=date(2026, 8, 3), + ) + retry_row = self._make_needs_review_run( + site=self.ground_site, window_start=date(2026, 8, 4), window_end=date(2026, 8, 4) + ) + + response = self.client.get(reverse('campaigns:approval_queue')) + + review_table = response.context['review_table'] + review_pks = {row.record.pk for row in review_table.rows} + self.assertNotIn(resolved_approved.pk, review_pks) + self.assertIn(retry_row.pk, review_pks) + + def test_unresolved_review_row_renders_live_search_widget_and_resolve_button(self): + run = self._make_needs_review_run(site_raw='F65') + + response = self.client.get(reverse('campaigns:approval_queue')) + + content = response.content.decode() + self.assertIn('Sites Needing Review', content) + self.assertIn('name="site_selection"', content) + self.assertIn(f'form="resolve-form-{run.pk}"', content) + self.assertIn('hx-get', content) + self.assertIn(reverse('campaigns:site_search'), content) + self.assertIn('input[this.value.length >= 2] changed delay:300ms', content) + self.assertIn('Create new Observatory', content) + self.assertIn('value="resolve_site"', content) + self.assertIn('btn-primary', content) + + def test_retry_row_renders_plain_text_site_and_resolve_button_no_input(self): + """Finding 8c: a run with site already set (flag still True) shows its resolved + site as plain text, no site-selection input, but still carries the Resolve button.""" + run = self._make_needs_review_run(site=self.ground_site) + + response = self.client.get(reverse('campaigns:approval_queue')) + + content = response.content.decode() + self.assertNotIn(f'id="site-input-{run.pk}"', content) + self.assertIn('FTS', content) + self.assertIn(f'id="resolve-form-{run.pk}"', content) + self.assertIn('value="resolve_site"', content) + + def test_review_table_empty_state_renders_configured_copy(self): + response = self.client.get(reverse('campaigns:approval_queue')) + self.assertContains(response, 'No sites currently need review.') + + +class TestIsPlaceholderObservatory(TestCase): + """Unit coverage for campaign_utils.is_placeholder_observatory() (22-06 Task 1) -- + the pure, DB-free string check both render_site() and _resolve_site() key off of.""" + + def test_placeholder_observatory_detected(self): + placeholder = Observatory.objects.create(obscode='DCT', name=f'{NEEDS_REVIEW_NAME_PREFIX}DCT', short_name='DCT') + self.assertTrue(campaign_utils.is_placeholder_observatory(placeholder)) + + def test_real_observatory_not_placeholder(self): + real = Observatory.objects.create(obscode='F65', name='Faulkes Telescope South', short_name='FTS') + self.assertFalse(campaign_utils.is_placeholder_observatory(real)) + + def test_none_is_not_placeholder(self): + self.assertFalse(campaign_utils.is_placeholder_observatory(None)) + + def test_tier3_create_uses_shared_prefix_constant(self): + """No behavioral change (Task 1): resolve_site()'s tier-3 fallback still produces + the exact same name shape, now built from NEEDS_REVIEW_NAME_PREFIX.""" + site, needs_review = resolve_site('ZZZ') + self.assertTrue(needs_review) + self.assertEqual(site.name, f'{NEEDS_REVIEW_NAME_PREFIX}ZZZ') + self.assertTrue(campaign_utils.is_placeholder_observatory(site)) + + +class TestSelectionToObscode(TestCase): + """Unit coverage for campaign_utils.selection_to_obscode() (debug/site-resolve-list-old-names). + + The site-search suggestion fragment (site_search_results.html) writes a COMBINED + ``'{display} ({obscode})'`` value into the site_selection input when a suggestion is + clicked. The approve/resolve handlers must map that combined form -- and a bare display + string or bare obscode typed/picked verbatim -- back to an obscode; anything unmappable + passes through unchanged (so resolve_site() can tier-1/2 it, or reject it). + """ + + POOL = { + 'G37': 'G37', + 'Lowell Discovery Telescope': 'G37', + 'G96': 'G96', + 'University of Arizona Mt. Lemmon Survey': 'G96', + } + + def setUp(self): + patcher = patch('solsys_code.campaign_utils.build_site_candidates', return_value=self.POOL) + patcher.start() + self.addCleanup(patcher.stop) + + def test_combined_display_obscode_widget_value_maps_to_obscode(self): + # The exact string site_search_results.html writes into the input on click. + self.assertEqual(campaign_utils.selection_to_obscode('Lowell Discovery Telescope (G37)'), 'G37') + + def test_bare_obscode_maps_via_exact_pool_hit(self): + self.assertEqual(campaign_utils.selection_to_obscode('G37'), 'G37') + + def test_bare_display_string_maps_via_exact_pool_hit(self): + self.assertEqual(campaign_utils.selection_to_obscode('University of Arizona Mt. Lemmon Survey'), 'G96') + + def test_display_containing_parens_keeps_only_trailing_obscode_group(self): + # A display that itself contains parentheses: only the LAST '(...)' is the obscode. + self.assertEqual(campaign_utils.selection_to_obscode('Weird (Annex) Site (G96)'), 'G96') + + def test_combined_form_with_unknown_display_falls_back_to_parenthesized_obscode(self): + # Display part not in the pool -> recover the parenthesized obscode token directly. + self.assertEqual(campaign_utils.selection_to_obscode('Some Brand New Scope (X99)'), 'X99') + + def test_unmappable_free_text_passes_through_unchanged(self): + self.assertEqual(campaign_utils.selection_to_obscode('Totally Unknown Site'), 'Totally Unknown Site') + + +class TestPlaceholderSiteReplacement(CampaignApprovalTestBase): + """22-06 gap closure (UAT gap 2B): a Sites Needing Review row whose site is a tier-3 + PLACEHOLDER Observatory now surfaces the correction widget (render_site()) and can be + replaced via resolve_site (view), while D-06 (never re-resolve a genuine site, racing + protection) and D-09 (never fabricate from unresolvable input) both stay intact. + + Fixture convention mirrors TestSitesNeedingReview: a Tier-1-resolvable ground + Observatory ('F65') so resolution never needs a live MPC API call. + """ + + @classmethod + def setUpTestData(cls) -> None: + super().setUpTestData() + cls.ground_site = Observatory.objects.create( + obscode='F65', + name='Faulkes Telescope South', + short_name='FTS', + lat=-31.2727, + lon=149.0644, + altitude=1149.0, + timezone='Australia/Sydney', + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + + def setUp(self): + self.client.login(username='staffcoordinator', password='pw') + + def _make_placeholder_observatory(self, obscode='DCT'): + """A tier-3 placeholder Observatory shaped exactly like resolve_site()'s fallback -- + NEEDS_REVIEW_NAME_PREFIX name, blank timezone (model default).""" + return Observatory.objects.create( + obscode=obscode, name=f'{NEEDS_REVIEW_NAME_PREFIX}{obscode}', short_name=obscode + ) + + def _make_placeholder_run(self, **overrides): + """An APPROVED run whose site is a placeholder Observatory (site_needs_review=True). + + Only creates its own default placeholder Observatory when the caller doesn't + already supply a ``site=`` override -- avoids creating a second, unwanted + placeholder (obscode/name collision) when the caller already made one. + """ + kwargs = { + 'approval_status': CampaignRun.ApprovalStatus.APPROVED, + 'site_needs_review': True, + } + if 'site' not in overrides: + kwargs['site'] = self._make_placeholder_observatory() + kwargs.update(overrides) + return self._make_pending_run(**kwargs) + + def test_placeholder_row_renders_live_search_widget_not_plain_text(self): + """render_site() (Task 2): a resolve-mode row whose site is a placeholder falls + through to the correction widget, distinguishing it from the genuine-site retry + state covered by test_retry_row_renders_plain_text_site_and_resolve_button_no_input.""" + run = self._make_placeholder_run(site_raw='DCT') + + response = self.client.get(reverse('campaigns:approval_queue')) + + content = response.content.decode() + self.assertIn(f'id="site-input-{run.pk}"', content) + self.assertIn('name="site_selection"', content) + self.assertIn(f'form="resolve-form-{run.pk}"', content) + self.assertIn('Create new Observatory', content) + + def test_placeholder_row_read_only_table_never_renders_widget(self): + """WR-01: even a placeholder-site row must never render the live widget in a + show_actions=False table, regardless of self.mode -- constructed directly since no + live show_actions=False + mode='resolve' view exists yet (a hypothetical future + read-only "resolved sites" audit view, per render_site()'s own docstring).""" + run = self._make_placeholder_run(site_raw='DCT', approval_status=CampaignRun.ApprovalStatus.APPROVED) + table = ApprovalQueueTable([run], show_actions=False, mode='resolve') + + rendered = table.render_site(run) + + self.assertNotIn('site_selection', str(rendered)) + self.assertIn('DCT', str(rendered)) + + def test_placeholder_replacement_repoints_site_and_clears_review_flag(self): + """Placeholder replacement: a real site_selection replaces the placeholder site and, + since preconditions are met (single-night window + telescope_instrument), the flag + clears and the calendar event projects.""" + placeholder = self._make_placeholder_observatory() + run = self._make_placeholder_run(site=placeholder, site_raw='DCT') + + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'F65'}, + follow=True, + ) + + self.assertEqual(response.status_code, 200) + run.refresh_from_db() + self.assertEqual(run.site_id, self.ground_site.pk) + self.assertFalse(run.site_needs_review) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 1) + messages_list = [str(m) for m in response.context['messages']] + self.assertIn('Site resolved — run added to the calendar.', messages_list) + + def test_placeholder_replacement_via_combined_widget_selection_resolves(self): + """Regression (debug/site-resolve-list-old-names): the site-search suggestion + fragment (site_search_results.html) writes the COMBINED ``'{display} ({obscode})'`` + string into the ``site_selection`` input when a staff member clicks a suggestion -- + e.g. ``'Faulkes Telescope South (F65)'``, exactly what the user selected for the + DCT/G37 row. The pre-fix handler mapped it via + ``build_site_candidates().get(selection, selection)``, whose pool has no key for the + combined form, so the 30+ char string reached ``resolve_site()`` and was rejected as + an oversized obscode (``len > _MAX_OBSCODE_LEN``) -> ``(None, True)`` -> the generic + "Could not resolve that site" error. ``selection_to_obscode()`` must now round-trip + the combined value back to F65 and replace the placeholder.""" + placeholder = self._make_placeholder_observatory() + run = self._make_placeholder_run(site=placeholder, site_raw='DCT') + # A controlled pool so the mapping is deterministic and no live MPC call is made -- + # keyed on the bare display string and bare obscode only (the real pool's shape), + # NOT the combined 'display (obscode)' form the widget actually submits. + pool = {'F65': 'F65', 'Faulkes Telescope South': 'F65', 'FTS': 'F65'} + with patch('solsys_code.campaign_utils.build_site_candidates', return_value=pool): + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'Faulkes Telescope South (F65)'}, + follow=True, + ) + + self.assertEqual(response.status_code, 200) + run.refresh_from_db() + self.assertEqual(run.site_id, self.ground_site.pk) + self.assertFalse(run.site_needs_review) + messages_list = [str(m) for m in response.context['messages']] + self.assertIn('Site resolved — run added to the calendar.', messages_list) + # The exact pre-fix failure must be gone. + self.assertNotIn( + 'Could not resolve that site. Try a different search term or an exact ' + 'MPC code, or use Create new Observatory.', + messages_list, + ) + + def test_placeholder_replacement_deletes_orphaned_placeholder_observatory(self): + """WR-03 (22-REVIEW.md re-review): once a placeholder Observatory is successfully + replaced and nothing else references it, the now-orphaned placeholder row itself + must be cleaned up -- not left behind to keep satisfying is_placeholder_observatory() + and polluting the CR-02 search-suggestion pool.""" + placeholder = self._make_placeholder_observatory() + run = self._make_placeholder_run(site=placeholder, site_raw='DCT') + + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'F65'}, + follow=True, + ) + + self.assertEqual(response.status_code, 200) + self.assertFalse(Observatory.objects.filter(pk=placeholder.pk).exists()) + + def test_placeholder_replacement_keeps_placeholder_still_referenced_by_another_run(self): + """WR-03: the orphaned-placeholder cleanup must never delete a placeholder still + referenced by a *different* CampaignRun (e.g. a second still-unresolved row sharing + the same not-yet-configured site).""" + placeholder = self._make_placeholder_observatory() + other_run = self._make_placeholder_run( + site=placeholder, site_raw='DCT', window_start=date(2026, 9, 1), window_end=date(2026, 9, 1) + ) + run = self._make_placeholder_run(site=placeholder, site_raw='DCT') + + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'F65'}, + follow=True, + ) + + self.assertEqual(response.status_code, 200) + self.assertTrue(Observatory.objects.filter(pk=placeholder.pk).exists()) + other_run.refresh_from_db() + self.assertEqual(other_run.site_id, placeholder.pk) + + def test_placeholder_replacement_failure_fabricates_no_second_placeholder(self): + """D-09: an unresolvable site_selection on a placeholder-site row must write + nothing new -- no second placeholder Observatory, the run keeps pointing at its + existing placeholder, and stays in Sites Needing Review.""" + placeholder = self._make_placeholder_observatory() + run = self._make_placeholder_run(site=placeholder, site_raw='DCT') + observatory_count_before = Observatory.objects.count() + + with patch( + 'solsys_code.campaign_utils.MPCObscodeFetcher.query', + side_effect=requests.exceptions.RequestException, + ): + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'NOWHERE'}, + follow=True, + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(Observatory.objects.count(), observatory_count_before) + run.refresh_from_db() + self.assertEqual(run.site_id, placeholder.pk) + self.assertTrue(run.site_needs_review) + messages_list = [str(m) for m in response.context['messages']] + self.assertIn( + 'Could not resolve that site. Try a different search term or an exact MPC code, ' + 'or use Create new Observatory.', + messages_list, + ) + + def test_genuine_site_still_never_re_resolved_when_replacing_placeholder_would_apply(self): + """D-06 preserved: a genuinely-resolved (non-placeholder) site is never re-resolved + by this same placeholder-replacement path -- resolve_site is not called, matching + the existing finding-8c coverage in TestSitesNeedingReview.""" + run = self._make_placeholder_run(site=self.ground_site, site_raw='F65') + with patch('solsys_code.campaign_views.resolve_site') as mock_resolve_site: + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site'}, + ) + self.assertEqual(response.status_code, 302) + mock_resolve_site.assert_not_called() + run.refresh_from_db() + self.assertEqual(run.site_id, self.ground_site.pk) + + def test_racing_second_resolve_after_placeholder_replacement_does_not_double_write(self): + """Racing-guard shape: once a placeholder has been replaced by a real site, a + second resolve_site POST for the same (now-real-site) run must not re-resolve -- + it falls straight to the never-re-resolve path (D-06), never a second write.""" + placeholder = self._make_placeholder_observatory() + run = self._make_placeholder_run(site=placeholder, site_raw='DCT') + + first_response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'F65'}, + follow=True, + ) + self.assertEqual(first_response.status_code, 200) + run.refresh_from_db() + self.assertEqual(run.site_id, self.ground_site.pk) + self.assertFalse(run.site_needs_review) + + # Simulate the retry surface: force the flag back on (as a failed-projection retry + # row would have it) without touching site, then re-POST resolve_site. + CampaignRun.objects.filter(pk=run.pk).update(site_needs_review=True) + with patch('solsys_code.campaign_views.resolve_site') as mock_resolve_site: + second_response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'F65'}, + follow=True, + ) + self.assertEqual(second_response.status_code, 200) + mock_resolve_site.assert_not_called() + run.refresh_from_db() + self.assertEqual(run.site_id, self.ground_site.pk) + + +class TestApprovalQueueSitesNeedingReviewGrouping(CampaignApprovalTestBase): + """UAT gap 2A closure (22-05): the Sites Needing Review section must be visually + differentiated from the historical Recently Decided table -- an actionable card, not + another plain DOM sibling -- while preserving D-07's locked document order (pending / + decided / sites-needing-review). This is presentation-only: no queryset/table/view + change, so these assertions hold even against empty tables. + """ + + def setUp(self): + self.client.login(username='staffcoordinator', password='pw') + + def test_sites_needing_review_renders_as_distinguishing_action_required_card(self): + response = self.client.get(reverse('campaigns:approval_queue')) + self.assertEqual(response.status_code, 200) + content = response.content.decode() + self.assertIn('border-warning', content) + self.assertIn('Sites Needing Review — action required', content) + + def test_d07_order_preserved_decided_precedes_sites_needing_review(self): + response = self.client.get(reverse('campaigns:approval_queue')) + content = response.content.decode() + decided_index = content.index('Recently Decided') + review_index = content.index('Sites Needing Review') + self.assertLess(decided_index, review_index) + self.assertIn('Pending Review', content) + self.assertIn('Recently Decided', content) + + +def _extract_create_observatory_form_fields(html_content: str, form_action_fragment: str) -> dict[str, str]: + """Extract ``name`` -> ``value`` for every ```` inside the + ``observatory_create.html`` form whose ``action`` attribute contains + ``form_action_fragment`` (CR-02). Stdlib ``html.parser.HTMLParser`` only -- no new + dependency. Used to replay ONLY the fields the rendered template itself contains, so a + round-trip test proves the template carries a field rather than merely that the view + logic works when handed a hand-built POST body.""" + + class _FormFieldParser(HTMLParser): + def __init__(self): + super().__init__() + self.in_target_form = False + self.fields: dict[str, str] = {} + + def handle_starttag(self, tag, attrs): + attrs_dict = dict(attrs) + if tag == 'form': + if form_action_fragment in (attrs_dict.get('action') or ''): + self.in_target_form = True + return + if tag == 'input' and self.in_target_form: + name = attrs_dict.get('name') + if name: + self.fields[name] = attrs_dict.get('value') or '' + + def handle_endtag(self, tag): + if tag == 'form' and self.in_target_form: + self.in_target_form = False + + parser = _FormFieldParser() + parser.feed(html_content) + return parser.fields + + +def _stub_to_observatory(): + """Create-and-return an Observatory row, mirroring what a real MPC-backed + ``MPCObscodeFetcher.to_observatory()`` call would do -- used to fake a successful + MPC lookup without hitting the live MPC API.""" + return Observatory.objects.create( + obscode='G37', + name='Lowell Discovery Telescope', + short_name='LDT', + lat=34.744, + lon=-111.4223, + altitude=2361.0, + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + + +class TestCreateObservatoryRoundTrip(CampaignApprovalTestBase): + """SITE-02/D-05: the "Create new Observatory" round-trip from the approval queue -- + ``?obscode=`` prefill and a validated ``?next=`` redirect back to the queue.""" + + def setUp(self): + self.client.login(username='staffcoordinator', password='pw') + self.next_url = reverse('campaigns:approval_queue') + + def test_get_with_obscode_and_next_prefills_form_initial(self): + create_url = reverse('solsys_code_observatory:create') + response = self.client.get(create_url, {'obscode': 'G37', 'next': self.next_url}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.context['form'].initial.get('obscode'), 'G37') + + def test_valid_create_with_safe_next_redirects_to_approval_queue(self): + create_url = reverse('solsys_code_observatory:create') + with ( + patch('solsys_code.solsys_code_observatory.views.MPCObscodeFetcher.query'), + patch( + 'solsys_code.solsys_code_observatory.views.MPCObscodeFetcher.to_observatory', + side_effect=_stub_to_observatory, + ), + ): + response = self.client.post(create_url, {'obscode': 'G37', 'next': self.next_url}) + self.assertRedirects(response, self.next_url) + self.assertEqual(Observatory.objects.filter(obscode='G37').count(), 1) + + def test_unsafe_next_falls_back_to_detail_redirect(self): + create_url = reverse('solsys_code_observatory:create') + with ( + patch('solsys_code.solsys_code_observatory.views.MPCObscodeFetcher.query'), + patch( + 'solsys_code.solsys_code_observatory.views.MPCObscodeFetcher.to_observatory', + side_effect=_stub_to_observatory, + ), + ): + response = self.client.post(create_url, {'obscode': 'G37', 'next': 'https://evil.example/steal'}) + observatory = Observatory.objects.get(obscode='G37') + self.assertRedirects(response, reverse('solsys_code_observatory:detail', kwargs={'pk': observatory.pk})) + + +class TestCreateObservatoryTemplateNextRoundTrip(CampaignApprovalTestBase): + """Permanent CR-02 regression (21-REVIEW-FIX.md / 21-VERIFICATION.md): the real + ``observatory_create.html`` template renders a hidden ``next`` input carrying + ``request.GET.next``, and POSTing ONLY the fields the rendered form itself contains + (extracted from the response HTML, not hand-constructed with ``next`` injected) + redirects to that ``next`` target. ``TestCreateObservatoryRoundTrip`` above proves the + view logic (``get_success_url()``) works when handed a manually-built POST body, but + never proves the template actually carries the field. The verifier independently + confirmed the real-template round-trip with a temporary end-to-end test (written, run, + then removed per verifier convention) before this class existed; this class makes that + coverage permanent. + """ + + def setUp(self): + self.client.login(username='staffcoordinator', password='pw') + self.next_url = reverse('campaigns:approval_queue') + self.create_url = reverse('solsys_code_observatory:create') + + def test_rendered_form_carries_next_field_and_replaying_it_redirects(self): + response = self.client.get(self.create_url, {'obscode': 'G37', 'next': self.next_url}) + self.assertEqual(response.status_code, 200) + + fields = _extract_create_observatory_form_fields(response.content.decode(), self.create_url) + # Load-bearing proof the template rendered the hidden field -- NOT hand-injected. + self.assertEqual(fields.get('next'), self.next_url) + + with ( + patch('solsys_code.solsys_code_observatory.views.MPCObscodeFetcher.query'), + patch( + 'solsys_code.solsys_code_observatory.views.MPCObscodeFetcher.to_observatory', + side_effect=_stub_to_observatory, + ), + ): + post_response = self.client.post(self.create_url, fields) + + self.assertRedirects(post_response, self.next_url) + self.assertEqual(Observatory.objects.filter(obscode='G37').count(), 1) + + +class TestCalendarNoChurn(CampaignApprovalTestBase): + """CAL-03: re-approve produces no duplicate event and no modified churn.""" + + @classmethod + def setUpTestData(cls) -> None: + super().setUpTestData() + # D-06: a Tier-1-resolvable ground Observatory for the default fixture's site_raw + # ('F65') so the first approve's calendar projection succeeds deterministically. + Observatory.objects.create( + obscode='F65', + name='Faulkes Telescope South', + short_name='FTS', + lat=-31.2727, + lon=149.0644, + altitude=1149.0, + timezone='Australia/Sydney', + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + + def setUp(self): + self.client.login(username='staffcoordinator', password='pw') + + def test_second_approve_leaves_event_count_and_modified_unchanged(self): + run = self._make_pending_run() + self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + event = CalendarEvent.objects.get(url=f'CAMPAIGN:{run.pk}') + modified_after_first_approve = event.modified + + # Second approve on an already-APPROVED row: updated_count == 0 (SUBMIT-03), so the + # projection block is never re-entered -- no duplicate, no modified churn. + self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 1) + event.refresh_from_db() + self.assertEqual(event.modified, modified_after_first_approve) + + +@override_settings(CACHES=ISOLATED_TEST_CACHES) +class TestSiteFuzzyMatch(TestCase): + """Wave-0 scaffold (SITE-01): cached MPC candidate pool + difflib fuzzy matching. + + Cache-isolated (bug #3, debug/site-search-degraded-pool-recurrence): this class calls the + REAL ``build_site_candidates()`` (writing the pool to the ``mpc_obscode_candidates`` cache + key) and ``cache.clear()`` in setUp/tearDown. Pinned to an in-memory LocMemCache so those + writes/clears never touch the shared /tmp file cache the dev runserver serves site-search + from -- without this, running this class wiped the runserver's warmed pool (the bug #3 + regression). + + Not-yet-existing helpers (``MPCObscodeFetcher.query_all``, ``campaign_utils. + build_site_candidates``, ``campaign_utils.fuzzy_match_candidates``) are referenced via + module attribute access so RED failures before Tasks 2-3 land are localized + AttributeErrors on these specific calls, not an ImportError that would de-collect the + whole test module (this class deliberately does not import the two campaign_utils + helpers by name at module scope). ``requests.get``/``MPCObscodeFetcher.query_all`` is + always mocked -- no test in this class hits the live MPC API. + + Uses a plain ``TestCase`` (not ``CampaignApprovalTestBase``) -- this class needs no + campaign/staff/pending-run fixtures, only ``Observatory`` rows for the local-pool + fallback case. + """ + + def setUp(self): + cache.clear() + + def tearDown(self): + cache.clear() + + @patch('requests.get') + def test_query_all_returns_fixture_dict_without_mutating_query_contract(self, mock_get): + mock_response = MagicMock(ok=True) + mock_response.json.return_value = BULK_MPC_FIXTURE + mock_get.return_value = mock_response + + fetcher = MPCObscodeFetcher() + result = fetcher.query_all() + + self.assertEqual(result, BULK_MPC_FIXTURE) + self.assertEqual(fetcher.obs_data, BULK_MPC_FIXTURE) + _, kwargs = mock_get.call_args + self.assertEqual(kwargs.get('json'), {}) + # query()'s own single-code contract is untouched by adding query_all -- a fresh + # fetcher's query() must still exist and behave independently of query_all(). + self.assertTrue(callable(fetcher.query)) + + @patch('solsys_code.solsys_code_observatory.utils.MPCObscodeFetcher.query_all') + def test_build_site_candidates_flattens_obscode_name_and_short_name(self, mock_query_all): + mock_query_all.return_value = BULK_MPC_FIXTURE + + pool = campaign_utils.build_site_candidates() + + for obscode, rec in BULK_MPC_FIXTURE.items(): + self.assertEqual(pool.get(obscode), obscode) + self.assertEqual(pool.get(rec['name_utf8']), obscode) + self.assertEqual(pool.get(rec['short_name']), obscode) + + @patch('solsys_code.solsys_code_observatory.utils.MPCObscodeFetcher.query_all') + def test_build_site_candidates_folds_list_valued_old_names(self, mock_query_all): + """Regression (debug/site-search-mpc-no-match): the live MPC bulk API returns + `old_names` as a JSON *list* (e.g. G96 -> ['Mt. Lemmon Survey']), not a string. The + pre-fix _flatten_mpc_candidates() used the list as a dict key/membership operand and + raised TypeError: unhashable type: 'list', which build_site_candidates() silently + swallowed -- discarding the ENTIRE MPC pool and degrading to local-only. Every MPC + record here has a real name and a list old_names; all must survive, and each prior + name must be an independently-matchable candidate mapped back to its obscode.""" + fixture = { + 'G96': { + 'name_utf8': 'University of Arizona Mt. Lemmon Survey', + 'short_name': 'University of Arizona Mt. Lemmon Survey', + 'old_names': ['Mt. Lemmon Survey'], + 'observations_type': 'optical', + 'longitude': 249.21128, + }, + '061': { + 'name_utf8': 'Uzhhorod', + 'short_name': 'Uzh', + 'old_names': ['Uzhgorod', 'Uzhorod'], + 'observations_type': 'optical', + 'longitude': 22.3, + }, + } + mock_query_all.return_value = fixture + + pool = campaign_utils.build_site_candidates() + + # The whole MPC pool survived (no swallowed TypeError -> no local-only degradation). + self.assertEqual(pool.get('G96'), 'G96') + self.assertEqual(pool.get('061'), '061') + self.assertEqual(pool.get('University of Arizona Mt. Lemmon Survey'), 'G96') + # Each list-valued old name is its own candidate, resolving back to the obscode. + self.assertEqual(pool.get('Mt. Lemmon Survey'), 'G96') + self.assertEqual(pool.get('Uzhgorod'), '061') + self.assertEqual(pool.get('Uzhorod'), '061') + # End-to-end: typing the historical name surfaces the current obscode as a suggestion. + matches = campaign_utils.substring_or_fuzzy_match_candidates('Mt. Lemmon', pool) + self.assertIn(('Mt. Lemmon Survey', 'G96'), matches) + + @patch('solsys_code.solsys_code_observatory.utils.MPCObscodeFetcher.query_all') + def test_flatten_mpc_candidates_tolerates_string_and_missing_old_names(self, mock_query_all): + """The list fix must not regress the string / None / missing old_names shapes that + the single-code query() endpoint and existing fixtures use.""" + fixture = { + 'AAA': {'name_utf8': 'Alpha Obs', 'short_name': 'Alpha', 'old_names': 'Legacy Alpha'}, + 'BBB': {'name_utf8': 'Beta Obs', 'short_name': 'Beta', 'old_names': None}, + 'CCC': {'name_utf8': 'Gamma Obs', 'short_name': 'Gamma'}, # old_names key absent + } + pool = campaign_utils._flatten_mpc_candidates(fixture) + + self.assertEqual(pool.get('Legacy Alpha'), 'AAA') # string old_names still folded in + self.assertEqual(pool.get('Beta Obs'), 'BBB') # None old_names simply skipped, no crash + self.assertEqual(pool.get('Gamma Obs'), 'CCC') # missing old_names key, no crash + + def test_flatten_mpc_candidates_survives_shape_surprise_in_any_field(self): + """Generalized robustness (debug/site-search-degraded-pool-recurrence, bug #3): the bug + #1 fix normalized only `old_names`, but the SAME failure family applies to every + candidate field. The live MPC bulk API has already shipped one field (`old_names`) as a + surprising JSON *list*; if `name_utf8`/`short_name` ever arrive non-str (or a whole + record is non-dict), the pre-hardening flatten would use the value in a dict-key / + membership operation and raise `TypeError: unhashable type: 'list'`, which + build_site_candidates() silently swallows -- dropping the ENTIRE ~2,712-code pool for + one malformed field. This feeds a fixture where EVERY candidate field takes each + surprising shape (list, dict, int, None, missing) and asserts (a) flatten never raises, + and (b) the one well-formed record still resolves -- so a single future shape surprise + degrades to "skip that field/record", never to a whole-pool drop that reverts live + site-search to 'No matches'. A live audit confirmed all 2,712 records currently carry + str name_utf8/short_name, so this is forward-looking hardening, not a current-data fix.""" + fixture = { + # A genuinely well-formed record must still survive alongside the malformed ones. + 'G37': {'name_utf8': 'Lowell Discovery Telescope', 'short_name': 'LDT', 'old_names': None}, + 'LST': {'name_utf8': ['a', 'list'], 'short_name': 'ListName', 'old_names': None}, # list name_utf8 + 'DCT': {'name_utf8': 'Dict Name', 'short_name': {'k': 'v'}, 'old_names': None}, # dict short_name + 'INT': {'name_utf8': 12345, 'short_name': 67890, 'old_names': None}, # int scalars + 'LON': {'name_utf8': 'Long Names Site', 'short_name': 'LNS', 'old_names': ['prior', ['nested']]}, + 'NON': 'this record is not a dict at all', # non-dict record + 'EMP': {}, # empty record, every candidate field missing + } + + # Must not raise despite every shape surprise above. + pool = campaign_utils._flatten_mpc_candidates(fixture) + + # The well-formed record and every well-formed field are intact. + self.assertEqual(pool.get('G37'), 'G37') + self.assertEqual(pool.get('Lowell Discovery Telescope'), 'G37') + self.assertEqual(pool.get('LDT'), 'G37') + # A non-str field is treated as absent, but the record's other good fields still fold in. + self.assertEqual(pool.get('ListName'), 'LST') # str short_name survives a list name_utf8 + self.assertEqual(pool.get('Dict Name'), 'DCT') # str name_utf8 survives a dict short_name + self.assertEqual(pool.get('Long Names Site'), 'LON') + self.assertEqual(pool.get('prior'), 'LON') # str element of a mixed old_names list folds in + # The obscode itself is always a (str) candidate, even when every scalar field is bad. + self.assertEqual(pool.get('INT'), 'INT') + self.assertEqual(pool.get('EMP'), 'EMP') + # No non-str value ever leaked in as a key (the exact unhashable-type crash vector). + self.assertTrue(all(isinstance(k, str) for k in pool)) + + @patch('solsys_code.solsys_code_observatory.utils.MPCObscodeFetcher.query_all') + def test_build_site_candidates_degraded_pool_uses_short_ttl(self, mock_query_all): + """A local-only fallback pool (MPC fetch failed) must be cached only briefly, so a + transient MPC outage cannot poison every site search for the full 24h TTL (the + amplifier that turned a swallowed error into a persistent, cross-restart outage).""" + mock_query_all.side_effect = requests.exceptions.RequestException + Observatory.objects.create( + obscode='Q64', name='El Sauce Observatory', short_name='El Sauce', lat=-30.47, lon=-70.77, altitude=1500.0 + ) + + with patch('solsys_code.campaign_utils.cache.set') as mock_set: + campaign_utils.build_site_candidates() + + _args, kwargs = mock_set.call_args + self.assertEqual(kwargs.get('timeout'), campaign_utils.MPC_CANDIDATE_FALLBACK_TTL_SECONDS) + self.assertLess( + campaign_utils.MPC_CANDIDATE_FALLBACK_TTL_SECONDS, campaign_utils.MPC_CANDIDATE_CACHE_TTL_SECONDS + ) + + @patch('solsys_code.solsys_code_observatory.utils.MPCObscodeFetcher.query_all') + def test_build_site_candidates_full_pool_uses_long_ttl(self, mock_query_all): + """The happy path (MPC fetch succeeded) still caches for the full 24h TTL.""" + mock_query_all.return_value = BULK_MPC_FIXTURE + + with patch('solsys_code.campaign_utils.cache.set') as mock_set: + campaign_utils.build_site_candidates() + + _args, kwargs = mock_set.call_args + self.assertEqual(kwargs.get('timeout'), campaign_utils.MPC_CANDIDATE_CACHE_TTL_SECONDS) + + @patch('solsys_code.solsys_code_observatory.utils.MPCObscodeFetcher.query_all') + def test_build_site_candidates_caches_result_under_fixed_key(self, mock_query_all): + mock_query_all.return_value = BULK_MPC_FIXTURE + + pool = campaign_utils.build_site_candidates() + + self.assertEqual(cache.get('mpc_obscode_candidates'), pool) + mock_query_all.assert_called_once() + + @patch('solsys_code.solsys_code_observatory.utils.MPCObscodeFetcher.query_all') + def test_build_site_candidates_second_call_reuses_cache_not_query_all(self, mock_query_all): + mock_query_all.return_value = BULK_MPC_FIXTURE + campaign_utils.build_site_candidates() + + # On the second call, a raise here would propagate if the cache were bypassed. + mock_query_all.side_effect = requests.exceptions.RequestException + pool_second = campaign_utils.build_site_candidates() + + self.assertIn('C65', pool_second) + mock_query_all.assert_called_once() + + @patch('solsys_code.solsys_code_observatory.utils.MPCObscodeFetcher.query_all') + def test_build_site_candidates_cold_cache_mpc_failure_falls_back_to_local_pool(self, mock_query_all): + mock_query_all.side_effect = requests.exceptions.RequestException + local = Observatory.objects.create( + obscode='Q64', + name='El Sauce Observatory', + short_name='El Sauce', + lat=-30.47, + lon=-70.77, + altitude=1500.0, + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + + pool = campaign_utils.build_site_candidates() + + self.assertEqual(pool.get(local.obscode), local.obscode) + self.assertEqual(pool.get(local.name), local.obscode) + + @patch('solsys_code.solsys_code_observatory.utils.MPCObscodeFetcher.query_all') + def test_build_site_candidates_excludes_placeholder_observatories(self, mock_query_all): + """CR-02 (22-REVIEW.md re-review): a tier-3 placeholder Observatory (name prefixed + NEEDS_REVIEW_NAME_PREFIX) must never surface as a search-suggestion candidate -- + neither by its obscode/short_name nor its 'NEEDS REVIEW: ...' display name -- or a + staff member could click it and have resolve_site() silently accept it as a genuine + resolution of itself.""" + mock_query_all.return_value = BULK_MPC_FIXTURE + Observatory.objects.create(obscode='DCT', name=f'{NEEDS_REVIEW_NAME_PREFIX}DCT', short_name='DCT') + + pool = campaign_utils.build_site_candidates() + + self.assertNotIn('DCT', pool) + self.assertNotIn(f'{NEEDS_REVIEW_NAME_PREFIX}DCT', pool) + + @patch('solsys_code.solsys_code_observatory.utils.MPCObscodeFetcher.query_all') + def test_fuzzy_match_candidates_exact_hit_includes_obscode(self, mock_query_all): + mock_query_all.return_value = BULK_MPC_FIXTURE + pool = campaign_utils.build_site_candidates() + + matches = campaign_utils.fuzzy_match_candidates('C65', pool) + + self.assertIn(('C65', 'C65'), matches) + + @patch('solsys_code.solsys_code_observatory.utils.MPCObscodeFetcher.query_all') + def test_fuzzy_match_candidates_near_typo_scores_above_cutoff(self, mock_query_all): + mock_query_all.return_value = BULK_MPC_FIXTURE + pool = campaign_utils.build_site_candidates() + + matches = campaign_utils.fuzzy_match_candidates('Siding Spring Observatry', pool) + + self.assertIn(('Siding Spring Observatory', 'W89'), matches) + + @patch('solsys_code.solsys_code_observatory.utils.MPCObscodeFetcher.query_all') + def test_fuzzy_match_candidates_nickname_returns_no_matches(self, mock_query_all): + """Pitfall 2: 'DCT' cannot bridge to 'Lowell Discovery Telescope' via difflib, even + against the widened pool -- the free-text/create-new fallback is load-bearing.""" + mock_query_all.return_value = BULK_MPC_FIXTURE + pool = campaign_utils.build_site_candidates() + + matches = campaign_utils.fuzzy_match_candidates('DCT', pool) + + self.assertEqual(matches, []) + + +@override_settings(CACHES=ISOLATED_TEST_CACHES) +class TestSiteSearchCacheIsolationRegression(TestCase): + """Regression (debug/site-search-degraded-pool-recurrence, bug #3): the campaign test suite + must NEVER read, write, or clear the shared FileBasedCache the dev runserver serves live + site-search from. + + Root cause of bug #3: settings.CACHES is a FileBasedCache at tempfile.gettempdir() (/tmp), + shared across processes, and Django does NOT swap the cache backend for tests the way it + swaps the database. So the campaign tests -- which call cache.clear() in setUp/tearDown and + the real build_site_candidates() (writing the 'mpc_obscode_candidates' key) -- were + read/writing/wiping the exact cache entry the runserver depends on. Running the suite to + verify a fix WIPED the runserver's warmed ~5,700-entry MPC candidate pool, so live + site-search reverted to "No matches" until the next successful cold rebuild. + + This test writes a sentinel directly into the real /tmp FileBasedCache (a fresh handle, + deliberately NOT the overridden default alias -- so it points at the runserver's actual + cache), then performs the exact cache operations the suite performs under + @override_settings(CACHES=ISOLATED_TEST_CACHES), and asserts the sentinel is untouched. + Without the isolation decorators this fails: cache.clear() on the shared FileBasedCache + wipes the sentinel (and the runserver's real pool) too. + """ + + def _runserver_file_cache(self): + # A direct FileBasedCache handle at the real settings location -- unaffected by this + # class's CACHES override (which only rebinds the `caches` registry / default proxy). + import tempfile + + from django.core.cache.backends.filebased import FileBasedCache + + return FileBasedCache(tempfile.gettempdir(), {}) + + def test_suite_cache_operations_do_not_touch_the_shared_runserver_file_cache(self): + import uuid + + runserver_cache = self._runserver_file_cache() + sentinel_key = f'runserver_warmed_pool_sentinel_{uuid.uuid4().hex}' + sentinel_value = {'G37': 'G37', 'Lowell Discovery Telescope': 'G37'} + runserver_cache.set(sentinel_key, sentinel_value, timeout=300) + try: + # Confirm the isolation override is actually in effect for the default cache the + # tests (and campaign_utils) use -- a LocMemCache, not the shared FileBasedCache. + from django.core.cache import caches + from django.core.cache.backends.locmem import LocMemCache + + self.assertIsInstance(caches['default'], LocMemCache) + + # The exact operations the campaign test classes perform every run: + cache.clear() # setUp/tearDown of TestSiteFuzzyMatch / ThrottleTest / SiteSearchViewTest + cache.set(campaign_utils._MPC_CANDIDATE_CACHE_KEY, {'X': 'X'}) # a real build_site_candidates() write + cache.set('site_search_throttle:1.2.3.4', 1) # a throttle write + + # Under isolation all of the above hit LocMemCache; the runserver's real /tmp pool + # is untouched. Without the isolation decorators, cache.clear() would have wiped it. + self.assertEqual(runserver_cache.get(sentinel_key), sentinel_value) + finally: + runserver_cache.delete(sentinel_key) + + +class TestApprovalQueueSiteSearchWidget(CampaignApprovalTestBase): + """D-10/22-REVIEWS.md findings 1 and 7: the approval queue's pending-row Site column is + a live-search widget (hx-get to campaigns:site_search), replacing the static datalist + from Plan 21-03, while keeping the "Create new Observatory" link and stored-XSS + escaping coverage. + + ``build_site_candidates`` is patched at the view's import site + (``solsys_code.campaign_views.build_site_candidates``) so every case here is + deterministic and never hits the live MPC API -- mirrors ``TestSiteFuzzyMatch``'s + mocking-boundary discipline, but at the view layer instead of the helper layer. + """ + + def setUp(self): + cache.clear() + self.client.login(username='staffcoordinator', password='pw') + self._candidate_pool = campaign_utils._flatten_mpc_candidates(BULK_MPC_FIXTURE) + patcher = patch('solsys_code.campaign_views.build_site_candidates', return_value=self._candidate_pool) + patcher.start() + self.addCleanup(patcher.stop) + + def tearDown(self): + cache.clear() + + def test_unresolved_pending_row_renders_live_search_widget_and_create_link(self): + run = self._make_pending_run(site=None, site_raw='F65', site_needs_review=False) + + response = self.client.get(reverse('campaigns:approval_queue')) + + content = response.content.decode() + self.assertIn('name="site_selection"', content) + self.assertIn(f'form="decide-form-{run.pk}"', content) + self.assertIn('hx-get', content) + self.assertIn(reverse('campaigns:site_search'), content) + # The raw corrected trigger string (unescaped -- it sits in the format_html + # literal, not a substituted attribute) -- 22-REVIEWS.md finding 1. + self.assertIn('input[this.value.length >= 2] changed delay:300ms', content) + self.assertNotIn('delay:300ms[', content) + self.assertIn(f'
tag may reach the response body.""" + self._make_pending_run(site=None, site_raw='', site_needs_review=False) + + response = self.client.get(reverse('campaigns:approval_queue')) + + content = response.content.decode() + self.assertNotIn('', content) + self.assertIn('<script>alert(1)</script>', content) + + def test_resolved_pending_row_renders_no_site_selection_input(self): + observatory = Observatory.objects.create( + obscode='F65', + name='Faulkes Telescope South', + short_name='FTS', + lat=-31.2727, + lon=149.0644, + altitude=1149.0, + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + run = self._make_pending_run(site=observatory, site_raw='F65', site_needs_review=False) + + response = self.client.get(reverse('campaigns:approval_queue')) + + content = response.content.decode() + self.assertNotIn(f'id="site-input-{run.pk}"', content) + self.assertIn('FTS', content) + + def test_decided_table_renders_no_site_selection_input(self): + self._make_pending_run( + site=None, + site_raw='F65', + site_needs_review=False, + approval_status=CampaignRun.ApprovalStatus.APPROVED, + ) + + response = self.client.get(reverse('campaigns:approval_queue')) + + self.assertNotIn('name="site_selection"', response.content.decode()) + + def test_pending_unresolved_row_renders_confirm_guard_and_flag(self): + """Quick task 260716-js7: a pending row whose site was never clicked from the + dropdown must carry both the known-resolved-tracking attribute/handler on the + input and the confirm-guard onclick on the Approve button, keyed on the row pk.""" + run = self._make_pending_run(site=None, site_raw='F65', site_needs_review=False) + + response = self.client.get(reverse('campaigns:approval_queue')) + + content = response.content.decode() + self.assertIn('data-site-resolved="false"', content) + self.assertIn('this.dataset.siteResolved', content) + self.assertIn(f"getElementById('site-input-{run.pk}')", content) + self.assertIn('dataset.siteResolved', content) + self.assertIn('does not look resolved yet', content) + + def test_decided_row_has_no_guard_and_no_flag(self): + """A read-only decided row (show_actions=False) must render neither the + known-resolved flag nor the confirm-guard message -- WR-01 read-only convention.""" + self._make_pending_run( + site=None, + site_raw='F65', + site_needs_review=False, + approval_status=CampaignRun.ApprovalStatus.APPROVED, + ) + + response = self.client.get(reverse('campaigns:approval_queue')) + + content = response.content.decode() + self.assertNotIn('data-site-resolved', content) + self.assertNotIn('does not look resolved yet', content) + + def test_suggestion_fragment_sets_known_resolved_flag(self): + """Clicking a suggestion in site_search_results.html must flip the paired + input's known-resolved flag so a following Approve click skips the confirm.""" + rendered = render_to_string( + 'campaigns/partials/site_search_results.html', + {'candidates': [('Lowell Discovery Telescope', 'G37')], 'input_id': 'site-input-1', 'query': 'low'}, + ) + + self.assertIn("inputEl.dataset.siteResolved = 'true';", rendered) + + +def _stub_i11_to_observatory(): + """Create-and-return a ground-based Gemini South Observatory row, mirroring what a + real MPC-backed ``MPCObscodeFetcher.to_observatory()`` call would do for the real + ``'I11'`` obscode -- used to fake a successful Tier-2 MPC lookup without hitting the + live MPC API. Cerro Pachón coordinates are realistic values (RESEARCH.md Code + Examples); exact lat/lon/altitude precision is not load-bearing for this test.""" + return Observatory.objects.create( + obscode='I11', + name='Gemini South Observatory, Cerro Pachon', + short_name='Gemini South', + lat=-30.2407, + lon=-70.7366, + altitude=2722.0, + timezone='America/Santiago', + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + + +class TestResolveSiteI11GeminiSouth(TestCase): + """D-06: ``resolve_site('I11')`` must resolve the real Gemini South Observatory as a + ground-based site with a real timezone via the resolver's ACTUAL Tier-2 single-code + path, with no manual admin edit needed and no live MPC network call. + + REVIEW finding #2 (HIGH, Codex): ``resolve_site()`` walks Tier 1 + (``Observatory.objects.get(obscode=code)``) then Tier 2 (single-code + ``MPCObscodeFetcher.query()`` -> ``to_observatory()`` at ``campaign_utils.py:184-200``) + -- it never calls ``query_all()``/``build_site_candidates()``, so a ``BULK_MPC_FIXTURE`` + ``'I11'`` entry (which only feeds the bulk fuzzy-match candidate-pool widget path) would + not influence this resolver at all, and is deliberately NOT added here. Instead this + mirrors the Phase 21 P04 precedent (STATE.md: "Mocked MPCObscodeFetcher.to_observatory() + directly (side_effect creating a real Observatory row), since to_observatory() reads + several MPC-response dict keys with no defaults and a bare query() mock would raise + MissingDataException"): both ``MPCObscodeFetcher.query`` (a no-op MagicMock, so the + Tier-2 network call never raises) and ``MPCObscodeFetcher.to_observatory`` (a + ``side_effect`` creating a real ground-based ``I11`` Observatory row) are patched. + """ + + def test_resolve_site_i11_resolves_gemini_south_ground_based(self): + with ( + patch('solsys_code.campaign_utils.MPCObscodeFetcher.query'), + patch( + 'solsys_code.campaign_utils.MPCObscodeFetcher.to_observatory', + side_effect=_stub_i11_to_observatory, + ), + ): + observatory, needs_review = resolve_site('I11') + + self.assertIsNotNone(observatory) + self.assertFalse(is_placeholder_observatory(observatory)) + self.assertFalse(observatory.name.startswith(NEEDS_REVIEW_NAME_PREFIX)) + self.assertIn('Gemini South', observatory.name) + self.assertIn('Gemini South', observatory.short_name) + self.assertEqual(observatory.observations_type, Observatory.OPTICAL_OBSTYPE) + self.assertEqual(observatory.timezone, 'America/Santiago') + self.assertFalse(needs_review) + self.assertEqual(Observatory.objects.filter(obscode='I11').count(), 1) + + +class TestGeminiFtScenario(CampaignApprovalTestBase): + """D-06/D-07: the real Gemini Fast-Turnaround GS-2026A-FT-115 informational run flows + through the SAME approve -> mark-status mechanism as any Magellan run, with no + special-casing. Its window is a 4-day range (2026-07-13..2026-07-16), so approving it + projects NO ``CAMPAIGN:{pk}`` ``CalendarEvent`` (range-window skip-by-design); marking + it weathered, then cancelled, must set ``run_status`` only and never crash or + fabricate an event (RESEARCH Pitfall 1, T-23-07). This scenario creates no new + production code -- it exercises Plan 02's already-built ``_set_run_status()`` + end-to-end against the real D-06 seed values. + """ + + @classmethod + def setUpTestData(cls) -> None: + super().setUpTestData() + cls.didymos_campaign = TargetList.objects.create(name='Didymos 2026') + # Tier-1-resolvable so approve's site resolution never needs a live MPC call -- + # Task 1 above (TestResolveSiteI11GeminiSouth) separately proves the Tier-2 + # resolver path for I11; this scenario's point is the run_status / + # no-event-fabrication mechanism, not re-proving site resolution. + cls.gemini_south = Observatory.objects.create( + obscode='I11', + name='Gemini South Observatory, Cerro Pachon', + short_name='Gemini South', + lat=-30.2407, + lon=-70.7366, + altitude=2722.0, + timezone='America/Santiago', + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + + def setUp(self): + self.client.login(username='staffcoordinator', password='pw') + + def test_gemini_ft115_range_window_flows_through_same_mechanism_no_event_fabricated(self): + run = self._make_pending_run( + campaign=self.didymos_campaign, + telescope_instrument='Gemini-South GMOS-S', + site_raw='I11', + window_start=date(2026, 7, 13), + window_end=date(2026, 7, 16), + contact_person='Thomas-Osip', + observation_details=( + 'GS-2026A-FT-115, 6.50 awarded hours (informational only -- not a real Gemini ODB submission)' + ), + target=None, + ) + + # (a) Approve: because the window is a 4-day range, no CalendarEvent is projected + # (range-window projection is skipped by design). + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(run.site_id, self.gemini_south.pk) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + + # (b)/(c)/(d) mark_weather_failure: normal redirect (no 500/IntegrityError), + # run_status set to WEATHER_TECH_FAILURE, still no CalendarEvent fabricated. + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'mark_weather_failure'} + ) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.run_status, CampaignRun.RunStatus.WEATHER_TECH_FAILURE) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + + # A follow-up mark_cancelled is a REAL transition (WEATHER_TECH_FAILURE -> + # CANCELLED are two distinct RunStatus values, not an idempotent no-op -- + # REVIEW finding, Codex MEDIUM); still no CalendarEvent fabricated. + response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'mark_cancelled'}) + self.assertEqual(response.status_code, 302) + run.refresh_from_db() + self.assertEqual(run.run_status, CampaignRun.RunStatus.CANCELLED) + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + + # Source assertion anchor (exact D-06 seed values, target left unset): + self.assertEqual(run.telescope_instrument, 'Gemini-South GMOS-S') + self.assertEqual(run.contact_person, 'Thomas-Osip') + self.assertEqual(run.campaign.name, 'Didymos 2026') + self.assertIsNone(run.target) diff --git a/solsys_code/tests/test_campaign_forms.py b/solsys_code/tests/test_campaign_forms.py new file mode 100644 index 00000000..5dc908dc --- /dev/null +++ b/solsys_code/tests/test_campaign_forms.py @@ -0,0 +1,184 @@ +"""Tests for `CampaignRunSubmissionForm` (SUBMIT-01/SUBMIT-04, D-05/D-06). + +Uses `TargetList.objects.create(...)` for the campaign fixture (never +`SiderealTargetFactory`/`Target` -- CLAUDE.md mandates non-sidereal-only fixtures for this +project; campaigns are `TargetList` objects, not `Target`, so no Target factory is needed here +at all). +""" + +from datetime import date + +from django import forms +from django.test import TestCase +from tom_targets.models import TargetList + +from solsys_code.campaign_forms import CampaignRunSubmissionForm + +CONTACT_PERSON = 'Jane Coordinator' +CONTACT_EMAIL = 'jane@example.org' + + +class CampaignRunSubmissionFormTest(TestCase): + """Behaviors from 16-01-PLAN.md Task 2 .""" + + @classmethod + def setUpTestData(cls) -> None: + cls.campaign = TargetList.objects.create(name='3I/ATLAS') + + def _minimal_data(self, **overrides): + data = { + 'campaign': self.campaign.pk, + 'contact_person': CONTACT_PERSON, + 'contact_email': CONTACT_EMAIL, + } + data.update(overrides) + return data + + def test_minimal_valid_submission(self): + """A form bound to only campaign/contact_person/contact_email (valid campaign pk) is valid.""" + form = CampaignRunSubmissionForm(data=self._minimal_data()) + self.assertTrue(form.is_valid(), form.errors) + + def test_missing_campaign_invalid(self): + """campaign is the only required model-backed field.""" + data = self._minimal_data() + del data['campaign'] + form = CampaignRunSubmissionForm(data=data) + self.assertFalse(form.is_valid()) + self.assertIn('campaign', form.errors) + + def test_missing_contact_person_invalid(self): + """contact_person is required at the form level (D-06), even though blank=True on the model.""" + data = self._minimal_data() + del data['contact_person'] + form = CampaignRunSubmissionForm(data=data) + self.assertFalse(form.is_valid()) + self.assertIn('contact_person', form.errors) + + def test_missing_contact_email_invalid(self): + """contact_email is required at the form level (D-06), even though blank=True on the model.""" + data = self._minimal_data() + del data['contact_email'] + form = CampaignRunSubmissionForm(data=data) + self.assertFalse(form.is_valid()) + self.assertIn('contact_email', form.errors) + + def test_honeypot_filled_still_valid(self): + """alt_contact_info is required=False; a filled honeypot does not fail validation (SUBMIT-04).""" + form = CampaignRunSubmissionForm(data=self._minimal_data(alt_contact_info='I am a bot')) + self.assertTrue(form.is_valid(), form.errors) + self.assertEqual(form.cleaned_data['alt_contact_info'], 'I am a bot') + + def test_honeypot_widget_is_hidden_input(self): + """alt_contact_info renders as a HiddenInput, never a normally-typed field (Pitfall 5).""" + form = CampaignRunSubmissionForm() + self.assertIsInstance(form.fields['alt_contact_info'].widget, forms.HiddenInput) + self.assertFalse(form.fields['alt_contact_info'].required) + + def test_telescope_instrument_not_required(self): + """Every field except campaign/contact_person/contact_email is optional (D-05).""" + form = CampaignRunSubmissionForm() + self.assertFalse(form.fields['telescope_instrument'].required) + + def test_contact_fields_required(self): + form = CampaignRunSubmissionForm() + self.assertTrue(form.fields['contact_person'].required) + self.assertTrue(form.fields['contact_email'].required) + + def test_site_raw_label_is_observing_site(self): + """site_raw's form label is 'Observing site', not the model verbose_name.""" + form = CampaignRunSubmissionForm() + self.assertEqual(form.fields['site_raw'].label, 'Observing site') + + def test_contact_public_opt_in_present_and_not_required(self): + """VIEW-05/D-07: the opt-in checkbox is not required, so an unchecked box validates.""" + form = CampaignRunSubmissionForm() + self.assertIn('contact_public_opt_in', form.fields) + self.assertFalse(form.fields['contact_public_opt_in'].required) + + def test_contact_public_opt_in_unchecked_defaults_false(self): + """VIEW-05: omitting the checkbox from POST data (unchecked box) cleans to False.""" + form = CampaignRunSubmissionForm(data=self._minimal_data()) + self.assertTrue(form.is_valid(), form.errors) + self.assertFalse(form.cleaned_data['contact_public_opt_in']) + + def test_contact_public_opt_in_checked_cleans_true(self): + """VIEW-05: submitting the checkbox as checked cleans to True.""" + form = CampaignRunSubmissionForm(data=self._minimal_data(contact_public_opt_in='on')) + self.assertTrue(form.is_valid(), form.errors) + self.assertTrue(form.cleaned_data['contact_public_opt_in']) + + def test_contact_public_opt_in_label(self): + form = CampaignRunSubmissionForm() + self.assertEqual(form.fields['contact_public_opt_in'].label, 'Show contact info publicly?') + + def test_is_plain_form_not_model_form(self): + """CampaignRunSubmissionForm must be a plain forms.Form, never a ModelForm (Pitfall 3).""" + self.assertTrue(issubclass(CampaignRunSubmissionForm, forms.Form)) + self.assertFalse(issubclass(CampaignRunSubmissionForm, forms.ModelForm)) + + +class CampaignRunSubmissionFormObsDateWindowTest(TestCase): + """260714-ilz: obs_date accepts flexible date/range text, parsed via parse_obs_window() + into cleaned_data['window_start']/['window_end'] (requirements 1/2/4/5/6). + """ + + @classmethod + def setUpTestData(cls) -> None: + cls.campaign = TargetList.objects.create(name='3I/ATLAS') + + def _minimal_data(self, **overrides): + data = { + 'campaign': self.campaign.pk, + 'contact_person': CONTACT_PERSON, + 'contact_email': CONTACT_EMAIL, + } + data.update(overrides) + return data + + def test_single_date_collapses_to_start_equals_end(self): + form = CampaignRunSubmissionForm(data=self._minimal_data(obs_date='2027-04-20')) + self.assertTrue(form.is_valid(), form.errors) + self.assertEqual(form.cleaned_data['window_start'], date(2027, 4, 20)) + self.assertEqual(form.cleaned_data['window_end'], date(2027, 4, 20)) + + def test_identical_double_hyphen_range_collapses_to_single_night(self): + """Requirement 4: an explicit start==end range still collapses to a single night.""" + form = CampaignRunSubmissionForm(data=self._minimal_data(obs_date='2027-04-20 -- 2027-04-20')) + self.assertTrue(form.is_valid(), form.errors) + self.assertEqual(form.cleaned_data['window_start'], date(2027, 4, 20)) + self.assertEqual(form.cleaned_data['window_end'], date(2027, 4, 20)) + + def test_identical_to_separated_range_collapses_to_single_night(self): + """The 'to'-separated equal-endpoint range exercises the second separator path.""" + form = CampaignRunSubmissionForm(data=self._minimal_data(obs_date='2027-04-20 to 2027-04-20')) + self.assertTrue(form.is_valid(), form.errors) + self.assertEqual(form.cleaned_data['window_start'], date(2027, 4, 20)) + self.assertEqual(form.cleaned_data['window_end'], date(2027, 4, 20)) + + def test_genuine_multi_night_range_is_valid(self): + """Requirement 1: a real multi-night range no longer hard-fails Django date validation.""" + form = CampaignRunSubmissionForm(data=self._minimal_data(obs_date='2027-04-20 -- 2027-05-11')) + self.assertTrue(form.is_valid(), form.errors) + self.assertEqual(form.cleaned_data['window_start'], date(2027, 4, 20)) + self.assertEqual(form.cleaned_data['window_end'], date(2027, 5, 11)) + + def test_blank_obs_date_is_valid_and_yields_tbd_window(self): + """Requirement 5: blank obs_date still produces a TBD run, both window fields None.""" + form = CampaignRunSubmissionForm(data=self._minimal_data()) + self.assertTrue(form.is_valid(), form.errors) + self.assertIsNone(form.cleaned_data['window_start']) + self.assertIsNone(form.cleaned_data['window_end']) + + def test_unparseable_obs_date_text_is_invalid_with_friendly_error(self): + """Requirement 2: genuinely unparseable non-blank text errors, never a silent TBD.""" + form = CampaignRunSubmissionForm(data=self._minimal_data(obs_date='sometime next spring')) + self.assertFalse(form.is_valid()) + self.assertIn('obs_date', form.errors) + self.assertTrue(form.errors['obs_date'][0]) + + def test_reversed_range_is_invalid_with_friendly_error(self): + """A reversed range (end < start) falls through to the unparseable-non-blank branch.""" + form = CampaignRunSubmissionForm(data=self._minimal_data(obs_date='2027-05-11 -- 2027-04-20')) + self.assertFalse(form.is_valid()) + self.assertIn('obs_date', form.errors) diff --git a/solsys_code/tests/test_campaign_gap.py b/solsys_code/tests/test_campaign_gap.py new file mode 100644 index 00000000..15e0cbfe --- /dev/null +++ b/solsys_code/tests/test_campaign_gap.py @@ -0,0 +1,616 @@ +"""Unit tests for the pure-computation coverage-gap module (GAP-02) + import guard (GAP-01). + +Depends only on `campaign_gap.py`, which itself depends only on `telescope_runs.sun_event` for +ephemerides -- never the heavy SPICE-loading ephemeris/views module. This module's own static +import-guard test mirrors the grep this file's plan verification step also runs, so the two stay +in agreement. + +Always uses `tom_targets.tests.factories.NonSiderealTargetFactory` for any Target fixture -- +never `SiderealTargetFactory` (CLAUDE.md: FOMO is exclusively for Solar System / non-sidereal +targets). +""" + +import inspect +from datetime import date, timedelta +from unittest import mock + +from django.core.cache import cache +from django.test import TestCase, override_settings +from django.urls import reverse +from tom_targets.models import TargetList +from tom_targets.tests.factories import NonSiderealTargetFactory + +from solsys_code import campaign_gap +from solsys_code.campaign_gap import ( + DEFAULT_WINDOW_DAYS, + MAX_WINDOW_DAYS, + build_gap_cache_key, + claimed_dates, + clamp_date_range, + observable_dates, +) +from solsys_code.models import CampaignRun +from solsys_code.solsys_code_observatory.models import Observatory + +TEST_CACHES = {'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} + + +class TestClampDateRange(TestCase): + """D-11: 90-day default window; 180-day hard cap; a smaller request is honoured.""" + + def test_default_window_is_90_days(self): + today = date(2026, 7, 4) + start, end = clamp_date_range(today, None) + self.assertEqual(start, today) + self.assertEqual(end, today + timedelta(days=DEFAULT_WINDOW_DAYS)) + + def test_far_future_end_clamps_to_180_days(self): + today = date(2026, 7, 4) + start, end = clamp_date_range(today, today + timedelta(days=500)) + self.assertEqual(start, today) + self.assertEqual(end, today + timedelta(days=MAX_WINDOW_DAYS)) + + def test_request_inside_cap_is_honoured(self): + today = date(2026, 7, 4) + start, end = clamp_date_range(today, today + timedelta(days=30)) + self.assertEqual(start, today) + self.assertEqual(end, today + timedelta(days=30)) + + +class TestBuildGapCacheKey(TestCase): + """D-10: cache key includes all four dimensions; null target encoded as 'none'.""" + + def test_key_contains_all_four_dimensions(self): + d0 = date(2026, 7, 4) + d1 = date(2026, 10, 2) + key = build_gap_cache_key(1, None, 5, d0, d1) + self.assertIn('1', key) + self.assertIn('none', key) + self.assertIn('5', key) + self.assertIn(d0.isoformat(), key) + self.assertIn(d1.isoformat(), key) + + def test_null_vs_real_target_do_not_collide(self): + d0 = date(2026, 7, 4) + d1 = date(2026, 10, 2) + key_none = build_gap_cache_key(1, None, 5, d0, d1) + key_real = build_gap_cache_key(1, 7, 5, d0, d1) + self.assertNotEqual(key_none, key_real) + + +class TestObservableDates(TestCase): + """D-03/D-04: non-zero dark window counts as observable; a ValueError date is skipped.""" + + @classmethod + def setUpTestData(cls): + cls.site = Observatory.objects.create( + obscode='268', + name='Las Campanas (Magellan-Clay)', + short_name='Magellan-Clay', + lon=-70.6926, + lat=-29.0146, + altitude=2402.0, + timezone='America/Santiago', + ) + + def test_returns_dates_with_nonzero_dark_window(self): + start = date(2026, 6, 10) + end = date(2026, 6, 12) + result = observable_dates(self.site, start, end) + # All 3 nights at a mid-latitude site should have a real dark window. + self.assertEqual(result, {start, start + timedelta(days=1), end}) + + def test_valueerror_date_is_skipped_loop_completes(self): + start = date(2026, 6, 10) + end = date(2026, 6, 12) + middle = start + timedelta(days=1) + + real_sun_event = campaign_gap.sun_event + + def flaky_sun_event(site, d, kind): + if d == middle: + raise ValueError('simulated unknown date') + return real_sun_event(site, d, kind) + + with mock.patch('solsys_code.campaign_gap.sun_event', side_effect=flaky_sun_event): + result = observable_dates(self.site, start, end) + + self.assertNotIn(middle, result) + self.assertIn(start, result) + self.assertIn(end, result) + + +@override_settings(CACHES=TEST_CACHES) +class TestClaimedDates(TestCase): + """D-05/D-08: window-range claiming, exclusions, and undated (TBD) flagging.""" + + @classmethod + def setUpTestData(cls): + cls.site = Observatory.objects.create( + obscode='269', + name='Las Campanas (Magellan-Baade)', + short_name='Magellan-Baade', + lon=-70.6926, + lat=-29.0146, + altitude=2402.0, + timezone='America/Santiago', + ) + cls.other_site = Observatory.objects.create( + obscode='809', + name='La Silla (NTT)', + short_name='NTT', + lon=-70.7345, + lat=-29.2567, + altitude=2400.0, + timezone='America/Santiago', + ) + cls.campaign = TargetList.objects.create(name='3I/ATLAS') + cls.target = NonSiderealTargetFactory.create() + cls.campaign.targets.add(cls.target) + + def setUp(self): + cache.clear() + + def _make_run(self, **kwargs): + defaults = { + 'campaign': self.campaign, + 'telescope_instrument': 'Magellan-Baade/IMACS', + 'site': self.site, + 'approval_status': CampaignRun.ApprovalStatus.APPROVED, + 'run_status': CampaignRun.RunStatus.OBSERVED, + } + defaults.update(kwargs) + return CampaignRun.objects.create(**defaults) + + def test_approved_run_claims_its_single_night_window(self): + night = date(2026, 7, 10) + self._make_run(window_start=night, window_end=night, telescope_instrument='A') + claimed, undated, unattributed, pending_narrowing = claimed_dates(self.campaign, self.target, self.site) + self.assertIn(night, claimed) + self.assertEqual(len(claimed), 1) + self.assertEqual(undated, []) + self.assertEqual(unattributed, []) + self.assertEqual(pending_narrowing, []) + + def test_range_run_claims_every_date_in_window(self): + window_start = date(2026, 8, 1) + window_end = date(2026, 8, 4) + self._make_run(window_start=window_start, window_end=window_end, telescope_instrument='RANGE') + claimed, undated, _, pending_narrowing = claimed_dates(self.campaign, self.target, self.site) + expected = { + date(2026, 8, 1), + date(2026, 8, 2), + date(2026, 8, 3), + date(2026, 8, 4), + } + self.assertEqual(claimed, expected) + self.assertEqual(undated, []) + # A ground run with a range never lands in pending_narrowing_runs -- that bucket + # is space-only (ASSET-02). + self.assertEqual(pending_narrowing, []) + + def test_cancelled_run_not_claimed(self): + night = date(2026, 7, 11) + self._make_run( + window_start=night, window_end=night, telescope_instrument='B', run_status=CampaignRun.RunStatus.CANCELLED + ) + claimed, _, _, _ = claimed_dates(self.campaign, self.target, self.site) + self.assertNotIn(night, claimed) + + def test_pending_review_run_not_claimed(self): + night = date(2026, 7, 12) + self._make_run( + window_start=night, + window_end=night, + telescope_instrument='C', + approval_status=CampaignRun.ApprovalStatus.PENDING_REVIEW, + ) + claimed, _, _, _ = claimed_dates(self.campaign, self.target, self.site) + self.assertNotIn(night, claimed) + + def test_undated_runs_flagged(self): + run = self._make_run(window_start=None, window_end=None, telescope_instrument='E') + claimed, undated, _, pending_narrowing = claimed_dates(self.campaign, self.target, self.site) + self.assertIn(run, undated) + self.assertNotIn(None, claimed) + # No date should have been added on behalf of this run. + self.assertEqual(len(claimed), 0) + # TBD runs never land in pending_narrowing_runs (D-09 explicit distinction). + self.assertEqual(pending_narrowing, []) + + def test_different_site_not_claimed(self): + night = date(2026, 7, 15) + self._make_run(window_start=night, window_end=night, telescope_instrument='F', site=self.other_site) + claimed, _, _, _ = claimed_dates(self.campaign, self.target, self.site) + self.assertNotIn(night, claimed) + + +@override_settings(CACHES=TEST_CACHES) +class TestClaimedDatesSpaceMission(TestCase): + """ASSET-01/ASSET-02/D-09: space-mission runs claim nothing until narrowed to a + single night; an un-narrowed range lands in pending_narrowing_runs, never + undated_runs; a TBD space-mission run lands in undated_runs, never + pending_narrowing_runs (D-09's explicit distinction).""" + + @classmethod + def setUpTestData(cls): + cls.space_site = Observatory.objects.create( + obscode='250', + name='Test Space Telescope', + short_name='TST', + observations_type=Observatory.SATELLITE_OBSTYPE, + ) + cls.campaign = TargetList.objects.create(name='Space Campaign') + cls.target = NonSiderealTargetFactory.create() + cls.campaign.targets.add(cls.target) + + def setUp(self): + cache.clear() + + def _make_run(self, **kwargs): + defaults = { + 'campaign': self.campaign, + 'telescope_instrument': 'HST/WFC3', + 'site': self.space_site, + 'approval_status': CampaignRun.ApprovalStatus.APPROVED, + 'run_status': CampaignRun.RunStatus.OBSERVED, + } + defaults.update(kwargs) + return CampaignRun.objects.create(**defaults) + + def test_narrowed_space_run_claims_its_single_night(self): + night = date(2026, 9, 1) + self._make_run(window_start=night, window_end=night, telescope_instrument='Narrowed') + claimed, undated, _, pending_narrowing = claimed_dates(self.campaign, self.target, self.space_site) + self.assertEqual(claimed, {night}) + self.assertEqual(undated, []) + self.assertEqual(pending_narrowing, []) + + def test_unnarrowed_space_run_claims_nothing_and_lands_in_pending_narrowing(self): + window_start = date(2026, 9, 1) + window_end = date(2026, 9, 10) + run = self._make_run(window_start=window_start, window_end=window_end, telescope_instrument='Unnarrowed') + claimed, undated, _, pending_narrowing = claimed_dates(self.campaign, self.target, self.space_site) + self.assertEqual(claimed, set()) + self.assertEqual(undated, []) + self.assertIn(run, pending_narrowing) + self.assertEqual(len(pending_narrowing), 1) + + def test_tbd_space_run_lands_in_undated_not_pending_narrowing(self): + run = self._make_run(window_start=None, window_end=None, telescope_instrument='TBD') + claimed, undated, _, pending_narrowing = claimed_dates(self.campaign, self.target, self.space_site) + self.assertEqual(claimed, set()) + self.assertIn(run, undated) + self.assertEqual(pending_narrowing, []) + + +@override_settings(CACHES=TEST_CACHES) +class TestClaimedDatesMultiTarget(TestCase): + """Pitfall 4: a multi-target campaign's target=None runs are unattributed, not counted.""" + + @classmethod + def setUpTestData(cls): + cls.site = Observatory.objects.create( + obscode='E10', + name='Siding Spring (FTS)', + short_name='FTS', + lon=149.0708, + lat=-31.2733, + altitude=1165.0, + timezone='Australia/Sydney', + ) + cls.campaign = TargetList.objects.create(name='Multi-target Campaign') + cls.target_a = NonSiderealTargetFactory.create() + cls.target_b = NonSiderealTargetFactory.create() + cls.campaign.targets.add(cls.target_a, cls.target_b) + + def setUp(self): + cache.clear() + + def test_target_none_run_is_unattributed_not_claimed_for_either_target(self): + night = date(2026, 7, 20) + CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='FTS/Sinistro', + site=self.site, + target=None, + window_start=night, + window_end=night, + approval_status=CampaignRun.ApprovalStatus.APPROVED, + run_status=CampaignRun.RunStatus.OBSERVED, + ) + claimed_a, _, unattributed_a, _ = claimed_dates(self.campaign, self.target_a, self.site) + claimed_b, _, unattributed_b, _ = claimed_dates(self.campaign, self.target_b, self.site) + self.assertNotIn(night, claimed_a) + self.assertNotIn(night, claimed_b) + self.assertEqual(len(unattributed_a), 1) + self.assertEqual(len(unattributed_b), 1) + + def test_target_specific_run_claimed_only_for_its_own_target(self): + night = date(2026, 7, 21) + CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='FTS/Sinistro-2', + site=self.site, + target=self.target_a, + window_start=night, + window_end=night, + approval_status=CampaignRun.ApprovalStatus.APPROVED, + run_status=CampaignRun.RunStatus.OBSERVED, + ) + claimed_a, _, _, _ = claimed_dates(self.campaign, self.target_a, self.site) + claimed_b, _, _, _ = claimed_dates(self.campaign, self.target_b, self.site) + self.assertIn(night, claimed_a) + self.assertNotIn(night, claimed_b) + + +@override_settings(CACHES=TEST_CACHES) +class TestGapAnalysisView(TestCase): + """Integration tests for CampaignGapAnalysisView (GAP-02): the fast table view never + triggers computation (D-09), a cache hit skips recomputation (D-10), out-of-scope + target/site pks are rejected server-side (T-17-01/Pitfall 3), and a single-target + campaign auto-selects its sole target (D-12). + """ + + @classmethod + def setUpTestData(cls): + cls.site = Observatory.objects.create( + obscode='097', + name='Wise Observatory', + short_name='Wise', + lon=34.7631, + lat=30.5958, + altitude=875.0, + timezone='Asia/Jerusalem', + ) + cls.other_site = Observatory.objects.create( + obscode='I33', + name='SOAR Cerro Pachon', + short_name='SOAR', + lon=-70.7342, + lat=-30.2379, + altitude=2738.0, + timezone='America/Santiago', + ) + + # Single-target campaign: gap_analysis_available is True (has a target + an approved + # run with a resolved site) -- used for the cache-hit and auto-select tests. + cls.target = NonSiderealTargetFactory.create() + cls.campaign = TargetList.objects.create(name='Single-target Campaign') + cls.campaign.targets.add(cls.target) + CampaignRun.objects.create( + campaign=cls.campaign, + telescope_instrument='Wise/LAST', + site=cls.site, + window_start=date(2026, 6, 1), + window_end=date(2026, 6, 1), + approval_status=CampaignRun.ApprovalStatus.APPROVED, + run_status=CampaignRun.RunStatus.OBSERVED, + ) + + # Multi-target campaign, with its own used site -- used for the IDOR tests. + cls.target_a = NonSiderealTargetFactory.create() + cls.target_b = NonSiderealTargetFactory.create() + cls.multi_campaign = TargetList.objects.create(name='Multi-target Campaign') + cls.multi_campaign.targets.add(cls.target_a, cls.target_b) + CampaignRun.objects.create( + campaign=cls.multi_campaign, + telescope_instrument='Wise/LAST-2', + site=cls.site, + window_start=date(2026, 6, 2), + window_end=date(2026, 6, 2), + approval_status=CampaignRun.ApprovalStatus.APPROVED, + run_status=CampaignRun.RunStatus.OBSERVED, + ) + + # A wholly separate campaign -- its target and site are never used by either + # campaign above (T-17-01/Pitfall 3 fixtures for the IDOR tests). + cls.foreign_target = NonSiderealTargetFactory.create() + cls.foreign_campaign = TargetList.objects.create(name='Foreign Campaign') + cls.foreign_campaign.targets.add(cls.foreign_target) + CampaignRun.objects.create( + campaign=cls.foreign_campaign, + telescope_instrument='SOAR/GHTS', + site=cls.other_site, + window_start=date(2026, 6, 3), + window_end=date(2026, 6, 3), + approval_status=CampaignRun.ApprovalStatus.APPROVED, + run_status=CampaignRun.RunStatus.OBSERVED, + ) + + def setUp(self): + cache.clear() + + def test_table_view_does_not_trigger_computation(self): + table_url = reverse('campaigns:table', kwargs={'pk': self.campaign.pk}) + with mock.patch('solsys_code.campaign_views.get_or_compute_gap') as mocked_gap: + response = self.client.get(table_url) + self.assertEqual(response.status_code, 200) + mocked_gap.assert_not_called() + + def test_cache_hit_skips_recomputation(self): + gap_url = reverse('campaigns:gap_analysis', kwargs={'pk': self.campaign.pk}) + end_date = date.today() + timedelta(days=1) + params = {'site': self.site.pk, 'end_date': end_date.isoformat()} + + # Mock sun_event (rather than the whole computation) so get_or_compute_gap's real + # cache-or-compute logic is genuinely exercised -- a fixed 2-day window (today, + # today+1) means exactly 2 sun_event calls total across both requests if (and only + # if) the second request is served entirely from cache. + with mock.patch('solsys_code.campaign_gap.sun_event', return_value=None) as mocked_sun_event: + response1 = self.client.get(gap_url, params) + response2 = self.client.get(gap_url, params) + + self.assertEqual(response1.status_code, 200) + self.assertEqual(response2.status_code, 200) + self.assertEqual(mocked_sun_event.call_count, 2) + self.assertEqual(response1.context['result']['computed_at'], response2.context['result']['computed_at']) + + def test_rejects_out_of_scope_target_and_site(self): + gap_url = reverse('campaigns:gap_analysis', kwargs={'pk': self.multi_campaign.pk}) + + with mock.patch('solsys_code.campaign_views.get_or_compute_gap') as mocked_gap: + response_bad_target = self.client.get(gap_url, {'target': self.foreign_target.pk, 'site': self.site.pk}) + response_bad_site = self.client.get(gap_url, {'target': self.target_a.pk, 'site': self.other_site.pk}) + + self.assertEqual(response_bad_target.status_code, 400) + self.assertEqual(response_bad_site.status_code, 400) + mocked_gap.assert_not_called() + + def test_single_target_autoselects(self): + gap_url = reverse('campaigns:gap_analysis', kwargs={'pk': self.campaign.pk}) + fixed_result = {'gap_dates': [], 'computed_at': 'sentinel'} + with mock.patch('solsys_code.campaign_views.get_or_compute_gap', return_value=fixed_result) as mocked_gap: + # No target_pk submitted -- the sole campaign target must still be used (D-12). + response = self.client.get(gap_url, {'site': self.site.pk}) + + self.assertEqual(response.status_code, 200) + mocked_gap.assert_called_once() + called_target = mocked_gap.call_args[0][1] + self.assertEqual(called_target, self.target) + + def test_pending_narrowing_alert_shown_for_unnarrowed_space_run(self): + """D-09: an un-narrowed space-mission run's page shows the distinct + pending-narrowing alert with its count. The space site has no timezone set, so + every date in the observable-dates loop raises ValueError and is skipped as + unknown (D-03) -- that's fine, the pending_narrowing_runs alert is driven purely + by claimed_dates() bucketing, independent of observable_dates().""" + space_site = Observatory.objects.create( + obscode='274', + name='Test Space Telescope 2', + short_name='TST2', + observations_type=Observatory.SATELLITE_OBSTYPE, + ) + target = NonSiderealTargetFactory.create() + campaign = TargetList.objects.create(name='Space Pending Campaign') + campaign.targets.add(target) + CampaignRun.objects.create( + campaign=campaign, + telescope_instrument='HST/WFC3', + site=space_site, + window_start=date(2026, 9, 1), + window_end=date(2026, 9, 10), + approval_status=CampaignRun.ApprovalStatus.APPROVED, + run_status=CampaignRun.RunStatus.OBSERVED, + ) + gap_url = reverse('campaigns:gap_analysis', kwargs={'pk': campaign.pk}) + + response = self.client.get(gap_url, {'site': space_site.pk}) + + self.assertEqual(response.status_code, 200) + self.assertContains(response, 'Pending narrowing: space-mission runs') + self.assertContains(response, '1 space-mission run(s)') + self.assertContains(response, "haven't narrowed to a") + + +@override_settings(CACHES=TEST_CACHES) +class TestGapAnalysisButton(TestCase): + """Integration tests for the 'Show Coverage Gaps' button's D-14 gating on the per-campaign + table page (GAP-02): enabled + linked when gap_analysis_available(), disabled with the + explanatory helper text otherwise -- proven at the rendered-template level, not just view + context (17-03-PLAN.md Task 2). + """ + + @classmethod + def setUpTestData(cls): + cls.site = Observatory.objects.create( + obscode='268', + name='Las Campanas (Magellan-Clay)', + short_name='Magellan-Clay', + lon=-70.6926, + lat=-29.0146, + altitude=2402.0, + timezone='America/Santiago', + ) + + def setUp(self): + cache.clear() + + def test_button_enabled_with_target_and_resolved_site(self): + target = NonSiderealTargetFactory.create() + campaign = TargetList.objects.create(name='Enabled Campaign') + campaign.targets.add(target) + CampaignRun.objects.create( + campaign=campaign, + telescope_instrument='Magellan-Clay/IMACS', + site=self.site, + window_start=date(2026, 7, 1), + window_end=date(2026, 7, 1), + approval_status=CampaignRun.ApprovalStatus.APPROVED, + run_status=CampaignRun.RunStatus.OBSERVED, + ) + table_url = reverse('campaigns:table', kwargs={'pk': campaign.pk}) + gap_url = reverse('campaigns:gap_analysis', kwargs={'pk': campaign.pk}) + + response = self.client.get(table_url) + + self.assertContains(response, 'Show Coverage Gaps') + self.assertContains(response, f'href="{gap_url}"') + self.assertNotContains( + response, + 'Coverage-gap analysis needs at least one campaign target and at least one run with a resolved site.', + ) + + def test_button_disabled_with_no_targets(self): + campaign = TargetList.objects.create(name='No-target Campaign') + # No .targets.add() -- zero targets, even though a resolved-site run exists, proving + # the gate is the target count, not merely "no runs at all" (D-14). + CampaignRun.objects.create( + campaign=campaign, + telescope_instrument='Magellan-Clay/IMACS', + site=self.site, + window_start=date(2026, 7, 2), + window_end=date(2026, 7, 2), + approval_status=CampaignRun.ApprovalStatus.APPROVED, + run_status=CampaignRun.RunStatus.OBSERVED, + ) + table_url = reverse('campaigns:table', kwargs={'pk': campaign.pk}) + gap_url = reverse('campaigns:gap_analysis', kwargs={'pk': campaign.pk}) + + response = self.client.get(table_url) + + self.assertContains( + response, + 'Coverage-gap analysis needs at least one campaign target and at least one run with a resolved site.', + ) + self.assertNotContains(response, f'href="{gap_url}"') + + def test_button_disabled_with_no_resolved_site(self): + target = NonSiderealTargetFactory.create() + campaign = TargetList.objects.create(name='No-site Campaign') + campaign.targets.add(target) + CampaignRun.objects.create( + campaign=campaign, + telescope_instrument='Unresolved/Site', + site=None, + site_raw='Some Unresolved Site', + window_start=date(2026, 7, 3), + window_end=date(2026, 7, 3), + approval_status=CampaignRun.ApprovalStatus.APPROVED, + run_status=CampaignRun.RunStatus.OBSERVED, + ) + table_url = reverse('campaigns:table', kwargs={'pk': campaign.pk}) + gap_url = reverse('campaigns:gap_analysis', kwargs={'pk': campaign.pk}) + + response = self.client.get(table_url) + + self.assertContains( + response, + 'Coverage-gap analysis needs at least one campaign target and at least one run with a resolved site.', + ) + self.assertNotContains(response, f'href="{gap_url}"') + + +class TestNoHeavyEphemerisImport(TestCase): + """GAP-01 (transitively): no phase module imports the heavy SPICE-loading ephemeris + module or `solsys_code.views` at module scope -- mirrors the plan's own verify grep.""" + + def test_campaign_gap_source_has_no_forbidden_imports(self): + source = inspect.getsource(campaign_gap) + for line in source.splitlines(): + stripped = line.strip() + self.assertFalse( + stripped.startswith(('from ', 'import ')) and 'ephem_utils' in stripped, + f'Forbidden ephem_utils import found: {line!r}', + ) + self.assertNotIn('from solsys_code.views import', stripped) diff --git a/solsys_code/tests/test_campaign_models.py b/solsys_code/tests/test_campaign_models.py new file mode 100644 index 00000000..a4e58f6b --- /dev/null +++ b/solsys_code/tests/test_campaign_models.py @@ -0,0 +1,238 @@ +from django.db import IntegrityError, transaction +from django.test import TestCase +from tom_targets.models import TargetList +from tom_targets.tests.factories import NonSiderealTargetFactory + +from solsys_code.models import CampaignRun + + +class TestCampaignRunFieldInventory(TestCase): + """CAMP-01: CampaignRun stores the full field inventory and re-fetches it correctly.""" + + @classmethod + def setUpTestData(cls) -> None: + cls.campaign = TargetList.objects.create(name='3I/ATLAS') + + def test_full_field_inventory_persists_and_reloads(self): + """SCHED-02: a single-night run (window_start == window_end) persists and reloads.""" + run = CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='FTN/MuSCAT3', + site_raw='F65', + site_needs_review=False, + window_start='2025-07-04', + window_end='2025-07-04', + filters_bandpass='griz', + observation_details='Photometric monitoring', + weather='Clear', + observation_outcome='Detected', + publication_plans='TBD', + open_to_collaboration=True, + comments='Nothing unusual', + contact_person='Test Person', + contact_email='test@example.com', + approval_status=CampaignRun.ApprovalStatus.APPROVED, + run_status=CampaignRun.RunStatus.OBSERVED, + ) + + reloaded = CampaignRun.objects.get(pk=run.pk) + + self.assertEqual(reloaded.campaign, self.campaign) + self.assertEqual(reloaded.telescope_instrument, 'FTN/MuSCAT3') + self.assertEqual(reloaded.site_raw, 'F65') + self.assertFalse(reloaded.site_needs_review) + self.assertEqual(str(reloaded.window_start), '2025-07-04') + self.assertEqual(reloaded.window_start, reloaded.window_end) + self.assertFalse(hasattr(reloaded, 'obs_date')) + self.assertFalse(hasattr(reloaded, 'ut_start')) + self.assertFalse(hasattr(reloaded, 'ut_end')) + self.assertEqual(reloaded.filters_bandpass, 'griz') + self.assertEqual(reloaded.observation_details, 'Photometric monitoring') + self.assertEqual(reloaded.weather, 'Clear') + self.assertEqual(reloaded.observation_outcome, 'Detected') + self.assertEqual(reloaded.publication_plans, 'TBD') + self.assertTrue(reloaded.open_to_collaboration) + self.assertEqual(reloaded.comments, 'Nothing unusual') + self.assertEqual(reloaded.contact_person, 'Test Person') + self.assertEqual(reloaded.contact_email, 'test@example.com') + self.assertEqual(reloaded.approval_status, CampaignRun.ApprovalStatus.APPROVED) + self.assertEqual(reloaded.run_status, CampaignRun.RunStatus.OBSERVED) + + +class TestCampaignRunWindowSchema(TestCase): + """SCHED-03/SCHED-04: TBD runs and the two partial UniqueConstraints.""" + + @classmethod + def setUpTestData(cls) -> None: + cls.campaign = TargetList.objects.create(name='3I/ATLAS') + + def test_tbd_run_saves_with_both_window_fields_null(self): + """SCHED-03: a fully-TBD run (both window fields null) persists and reloads.""" + run = CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='JWST', + window_start=None, + window_end=None, + contact_person='Carrie Holt', + ) + + reloaded = CampaignRun.objects.get(pk=run.pk) + + self.assertIsNone(reloaded.window_start) + self.assertIsNone(reloaded.window_end) + + def test_tbd_same_contact_person_collides(self): + """SCHED-04 (TBD branch): same campaign+telescope_instrument+contact_person collides.""" + CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='JWST', + window_start=None, + window_end=None, + contact_person='Carrie Holt', + ) + + with self.assertRaises(IntegrityError): + with transaction.atomic(): + CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='JWST', + window_start=None, + window_end=None, + contact_person='Carrie Holt', + ) + + def test_tbd_differing_contact_person_both_save(self): + """SCHED-04 (TBD branch): contact_person discriminates otherwise-identical TBD rows.""" + CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='JWST', + window_start=None, + window_end=None, + contact_person='Carrie Holt', + ) + with transaction.atomic(): + second = CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='JWST', + window_start=None, + window_end=None, + contact_person='Martin Cordiner', + ) + + self.assertIsNotNone(second.pk) + self.assertEqual(CampaignRun.objects.filter(telescope_instrument='JWST').count(), 2) + + def test_resolved_window_same_key_collides(self): + """SCHED-04 (resolved branch): same campaign+telescope_instrument+window_start+window_end collides.""" + CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='FTN/MuSCAT3', + window_start='2025-07-04', + window_end='2025-07-04', + ) + + with self.assertRaises(IntegrityError): + with transaction.atomic(): + CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='FTN/MuSCAT3', + window_start='2025-07-04', + window_end='2025-07-04', + ) + + def test_mismatched_window_start_end_pair_rejected_by_db(self): + """WR-02: window_start/window_end must be null together at the DB level.""" + with self.assertRaises(IntegrityError): + with transaction.atomic(): + CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='FTN/MuSCAT3', + window_start='2025-07-04', + window_end=None, + ) + + +class TestCampaignRunOptionalTarget(TestCase): + """CAMP-02: target is nullable; single-target campaigns work without ever setting it.""" + + @classmethod + def setUpTestData(cls) -> None: + cls.campaign = TargetList.objects.create(name='3I/ATLAS') + + def test_campaign_run_without_target_persists_and_reloads(self): + run = CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='VLT/MUSE', + ) + + reloaded = CampaignRun.objects.get(pk=run.pk) + + self.assertIsNone(reloaded.target) + + def test_campaign_run_with_linked_target_persists_and_reloads(self): + target = NonSiderealTargetFactory.create() + self.campaign.targets.add(target) + + run = CampaignRun.objects.create( + campaign=self.campaign, + target=target, + telescope_instrument='FTN/MuSCAT3', + ) + + reloaded = CampaignRun.objects.get(pk=run.pk) + + self.assertEqual(reloaded.target, target) + + +class TestCampaignRunStatusVocabulary(TestCase): + """CAMP-03: two-field status with correct defaults and controlled vocabulary sizes.""" + + @classmethod + def setUpTestData(cls) -> None: + cls.campaign = TargetList.objects.create(name='3I/ATLAS') + + def test_default_statuses_on_fresh_campaign_run(self): + run = CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='FTN/MuSCAT3', + ) + + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.PENDING_REVIEW) + self.assertEqual(run.run_status, CampaignRun.RunStatus.REQUESTED) + + def test_approval_status_has_exactly_three_members(self): + self.assertEqual(len(CampaignRun.ApprovalStatus.choices), 3) + + def test_run_status_has_exactly_eight_members(self): + self.assertEqual(len(CampaignRun.RunStatus.choices), 8) + + +class TestCampaignRunWindowNeedsReviewFields(TestCase): + """IMPORT-01/IMPORT-02: original_obs_date_raw/window_needs_review defaults and persistence.""" + + @classmethod + def setUpTestData(cls) -> None: + cls.campaign = TargetList.objects.create(name='3I/ATLAS') + + def test_defaults_on_fresh_campaign_run(self): + run = CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='FTN/MuSCAT3', + ) + + self.assertEqual(run.original_obs_date_raw, '') + self.assertFalse(run.window_needs_review) + + def test_fields_are_assignable_and_persist(self): + run = CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='JWST', + contact_person='Carrie Holt', + original_obs_date_raw='TBD pending Cycle 2', + window_needs_review=True, + ) + + reloaded = CampaignRun.objects.get(pk=run.pk) + + self.assertEqual(reloaded.original_obs_date_raw, 'TBD pending Cycle 2') + self.assertTrue(reloaded.window_needs_review) diff --git a/solsys_code/tests/test_campaign_site_search.py b/solsys_code/tests/test_campaign_site_search.py new file mode 100644 index 00000000..4d1a5d76 --- /dev/null +++ b/solsys_code/tests/test_campaign_site_search.py @@ -0,0 +1,291 @@ +"""Tests for the shared anonymous live-search endpoint (Phase 22 Plan 01, D-01..D-05). + +Covers the new ``substring_or_fuzzy_match_candidates()`` matcher (D-04/D-05) and the +per-IP throttle helper (D-02) in ``campaign_utils.py``, plus the ``SiteSearchView`` +endpoint itself (D-01/D-03) -- anonymous access, HTML-fragment response shape, the +2-char minimum-length gate (22-REVIEWS.md finding 4), and the ``input_id`` server-side +allowlist + JS-string escaping (22-REVIEWS.md finding 2). + +Reuses ``BULK_MPC_FIXTURE``/``campaign_utils._flatten_mpc_candidates()`` from +``test_campaign_approval.py`` (same pool-building convention already established there). +""" + +import difflib +from unittest.mock import patch + +from django.contrib.auth.models import User +from django.core.cache import cache +from django.test import TestCase, override_settings +from django.urls import reverse + +from solsys_code import campaign_utils +from solsys_code.campaign_utils import ( + _check_and_increment_throttle, + fuzzy_match_candidates, + substring_or_fuzzy_match_candidates, +) +from solsys_code.tests.test_campaign_approval import BULK_MPC_FIXTURE, ISOLATED_TEST_CACHES + + +class SubstringOrFuzzyMatchCandidatesTest(TestCase): + """D-04: substring-first containment matching, difflib fallback only on zero hits.""" + + def setUp(self): + self.pool = dict(campaign_utils._flatten_mpc_candidates(BULK_MPC_FIXTURE)) + # BULK_MPC_FIXTURE only has one Faulkes site (F65); add a second Faulkes-family + # record here (operator's motivating "faulkes surfaces both" example) since + # BULK_MPC_FIXTURE deliberately only fixtures one. + self.pool['Haleakala-Faulkes Telescope North'] = 'F65N' + + def test_substring_hit_surfaces_all_faulkes_candidates(self): + results = substring_or_fuzzy_match_candidates('faulkes', self.pool) + displays = [display for display, _obscode in results] + self.assertGreaterEqual(len(results), 2) + self.assertIn('Faulkes Telescope South', displays) + self.assertIn('Haleakala-Faulkes Telescope North', displays) + for display in displays: + self.assertIn('faulkes', display.lower()) + + def test_case_insensitive_same_result_set(self): + lower = substring_or_fuzzy_match_candidates('faulkes', self.pool) + upper = substring_or_fuzzy_match_candidates('FAULKES', self.pool) + self.assertEqual(set(lower), set(upper)) + + def test_substring_beats_difflib_cutoff_for_lowell(self): + results = substring_or_fuzzy_match_candidates('lowell', self.pool) + displays = [display for display, _obscode in results] + self.assertIn('Lowell Discovery Telescope', displays) + # Prove difflib alone (at its 0.6 cutoff) would NOT bridge this short partial + # query against the long official MPC string -- substring containment is what + # actually finds it here, not the fallback. + difflib_only = difflib.get_close_matches('lowell', self.pool.keys(), n=5, cutoff=0.6) + self.assertNotIn('Lowell Discovery Telescope', difflib_only) + + def test_difflib_fallback_only_when_containment_finds_nothing(self): + pool = {'Cassini Occultation Station': 'X01'} + query = 'Cassini Ocultation Station' # typo: missing one 'c', no substring hit + self.assertNotIn(query.lower(), 'cassini occultation station') + results = substring_or_fuzzy_match_candidates(query, pool) + self.assertEqual(results, [('Cassini Occultation Station', 'X01')]) + + def test_blank_or_whitespace_input_returns_empty_list(self): + self.assertEqual(substring_or_fuzzy_match_candidates('', self.pool), []) + self.assertEqual(substring_or_fuzzy_match_candidates(' ', self.pool), []) + + def test_limit_caps_result_length(self): + pool = {f'Site Alpha {i}': f'A{i:02d}' for i in range(20)} + default_capped = substring_or_fuzzy_match_candidates('site alpha', pool) + self.assertEqual(len(default_capped), 8) + explicit_capped = substring_or_fuzzy_match_candidates('site alpha', pool, limit=3) + self.assertEqual(len(explicit_capped), 3) + + def test_fuzzy_match_candidates_unaffected_by_new_n_parameter_default(self): + # fuzzy_match_candidates() itself must remain behaviorally unchanged for its + # existing single call site (ApprovalQueueTable.render_site()) -- default n=5. + pool = {f'Site Alpha {i}': f'A{i:02d}' for i in range(20)} + results = fuzzy_match_candidates('Site Alpha', pool) + self.assertLessEqual(len(results), 5) + + +class WhitespaceVariantDedupTest(TestCase): + """debug/duplicate-mpc-candidate-match: an ``old_names`` whitespace-variant of a record's + current name must NOT surface the same site twice in the suggestion dropdown. + + The live MPC bulk data carries, for 32 of 2,712 records, an ``old_names`` entry that is + the current name with extra internal/trailing whitespace (e.g. Z23's ``name_utf8`` + ``'Nordic Optical Telescope, La Palma'`` vs its ``old_names`` ``'Nordic Optical + Telescope, La Palma'``). Those two byte-distinct strings render byte-for-byte + identically in HTML, so before the fix both appeared as separate, identical suggestion + rows. The candidate pool now dedups on the whitespace-normalized (visible-rendered) form. + """ + + # Z23-shaped: name_utf8/short_name single-spaced, old_names with a 5-space run -- the + # exact live shape captured in the debug session's reproduction. + Z23_INTERNAL_SPACE_FIXTURE = { + 'Z23': { + 'name_utf8': 'Nordic Optical Telescope, La Palma', + 'short_name': 'Nordic Optical Telescope, La Palma', + 'old_names': ['Nordic Optical Telescope, La Palma'], + 'observations_type': 'optical', + 'longitude': 342.11492, + }, + } + + # The far more common live shape: old_names is the current name with a trailing space + # (e.g. obscode 434 'S. Benedetto Po ' vs 'S. Benedetto Po'). + TRAILING_SPACE_FIXTURE = { + '434': { + 'name_utf8': 'S. Benedetto Po', + 'short_name': 'S. Benedetto Po', + 'old_names': ['S. Benedetto Po '], + 'observations_type': 'fixed', + 'longitude': 10.9, + }, + } + + def test_flatten_collapses_internal_whitespace_variant_to_one_candidate(self): + flat = campaign_utils._flatten_mpc_candidates(self.Z23_INTERNAL_SPACE_FIXTURE) + name_keys = [k for k, code in flat.items() if code == 'Z23' and 'nordi' in k.lower()] + self.assertEqual(name_keys, ['Nordic Optical Telescope, La Palma']) + + def test_flatten_collapses_trailing_whitespace_variant_to_one_candidate(self): + flat = campaign_utils._flatten_mpc_candidates(self.TRAILING_SPACE_FIXTURE) + name_keys = [k for k, code in flat.items() if code == '434' and 'benedetto' in k.lower()] + self.assertEqual(name_keys, ['S. Benedetto Po']) + + def test_search_returns_single_suggestion_for_whitespace_variant_record(self): + pool = campaign_utils._flatten_mpc_candidates(self.Z23_INTERNAL_SPACE_FIXTURE) + results = substring_or_fuzzy_match_candidates('Nordi', pool) + self.assertEqual(results, [('Nordic Optical Telescope, La Palma', 'Z23')]) + + def test_normalize_candidate_matches_html_whitespace_collapsing(self): + normalize = campaign_utils._normalize_candidate + self.assertEqual(normalize('Nordic Optical, La Palma'), 'Nordic Optical, La Palma') + self.assertEqual(normalize('S. Benedetto Po '), 'S. Benedetto Po') + self.assertEqual(normalize(' a\t b\n c '), 'a b c') + + +@override_settings(CACHES=ISOLATED_TEST_CACHES) +class ThrottleTest(TestCase): + """D-02: per-IP fixed-window throttle via django.core.cache. + + Cache-isolated (bug #3, debug/site-search-degraded-pool-recurrence): writes throttle keys + and calls ``cache.clear()`` -- pinned to an in-memory LocMemCache so it never wipes the + shared /tmp file cache the dev runserver serves site-search from. + """ + + def setUp(self): + cache.clear() + + @patch.object(campaign_utils, 'SITE_SEARCH_THROTTLE_LIMIT', 3) + def test_allows_up_to_limit_then_rejects(self): + for _ in range(3): + self.assertTrue(_check_and_increment_throttle('1.2.3.4')) + self.assertFalse(_check_and_increment_throttle('1.2.3.4')) + + @patch.object(campaign_utils, 'SITE_SEARCH_THROTTLE_LIMIT', 3) + def test_counts_different_ips_independently(self): + for _ in range(3): + self.assertTrue(_check_and_increment_throttle('1.1.1.1')) + self.assertFalse(_check_and_increment_throttle('1.1.1.1')) + self.assertTrue(_check_and_increment_throttle('2.2.2.2')) + + +@override_settings(CACHES=ISOLATED_TEST_CACHES) +class SiteSearchViewTest(TestCase): + """D-01/D-03: anonymous, throttled, HTML-fragment live-search endpoint. + + Cache-isolated (bug #3, debug/site-search-degraded-pool-recurrence): calls ``cache.clear()`` + in setUp and exercises the throttle -- pinned to an in-memory LocMemCache so it never wipes + the shared /tmp file cache the dev runserver serves site-search from. + """ + + @classmethod + def setUpTestData(cls): + cls.staff_user = User.objects.create_user(username='staffcoordinator', password='pw', is_staff=True) + + def setUp(self): + cache.clear() + patcher = patch( + 'solsys_code.campaign_views.build_site_candidates', + return_value=campaign_utils._flatten_mpc_candidates(BULK_MPC_FIXTURE), + ) + self.mock_build_site_candidates = patcher.start() + self.addCleanup(patcher.stop) + + def test_anonymous_get_returns_html_fragment_with_suggestion(self): + response = self.client.get(reverse('campaigns:site_search'), {'q': 'faulkes'}) + self.assertEqual(response.status_code, 200) + self.assertTrue(response['Content-Type'].startswith('text/html')) + self.assertContains(response, '` -- htmx's hx-get sends `?site_raw=`, never `q`. + response = self.client.get(reverse('campaigns:site_search'), {'site_raw': 'faulkes'}) + self.assertEqual(response.status_code, 200) + self.assertContains(response, '` -- same missing-`q` defect. + response = self.client.get( + reverse('campaigns:site_search'), {'site_selection': 'faulkes', 'input_id': 'site-input-1'} + ) + self.assertEqual(response.status_code, 200) + self.assertContains(response, ' None: + cls.campaign = TargetList.objects.create(name='3I/ATLAS') + cls.staff_with_email = User.objects.create_user( + username='staffwithemail', password='pw', is_staff=True, email='staff@example.org' + ) + cls.staff_blank_email = User.objects.create_user( + username='staffblankemail', password='pw', is_staff=True, email='' + ) + cls.non_staff_user = User.objects.create_user( + username='regularuser', password='pw', is_staff=False, email='regular@example.org' + ) + + def submit_url(self): + return reverse('campaigns:submit') + + def thanks_url(self): + return reverse('campaigns:submission_thanks') + + def minimal_valid_data(self, **overrides): + data = { + 'campaign': self.campaign.pk, + 'contact_person': CONTACT_PERSON, + 'contact_email': CONTACT_EMAIL, + } + data.update(overrides) + return data + + +class TestCampaignSubmission(CampaignSubmissionTestBase): + """SUBMIT-01: minimal valid submission creates a PENDING_REVIEW CampaignRun.""" + + def test_minimal_valid_submission_creates_pending_run(self): + response = self.client.post(self.submit_url(), data=self.minimal_valid_data(obs_date=OBS_DATE.isoformat())) + self.assertEqual(CampaignRun.objects.count(), 1) + run = CampaignRun.objects.get() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.PENDING_REVIEW) + self.assertEqual(run.campaign, self.campaign) + self.assertEqual(run.contact_person, CONTACT_PERSON) + self.assertEqual(run.contact_email, CONTACT_EMAIL) + # SCHED-02: the form's single observing-date field collapses to window_start == + # window_end (a single-night run). + self.assertEqual(run.window_start, OBS_DATE) + self.assertEqual(run.window_end, OBS_DATE) + self.assertRedirects(response, self.thanks_url()) + + def test_contact_public_opt_in_checked_persists_true(self): + """VIEW-05: submitting the box checked persists contact_public_opt_in=True.""" + self.client.post(self.submit_url(), data=self.minimal_valid_data(contact_public_opt_in='on')) + run = CampaignRun.objects.get() + self.assertTrue(run.contact_public_opt_in) + + def test_contact_public_opt_in_unchecked_persists_false(self): + """VIEW-05: an unchecked box (default opt-out) persists contact_public_opt_in=False.""" + self.client.post(self.submit_url(), data=self.minimal_valid_data()) + run = CampaignRun.objects.get() + self.assertFalse(run.contact_public_opt_in) + + def test_get_returns_200_and_renders_form(self): + response = self.client.get(self.submit_url()) + self.assertEqual(response.status_code, 200) + self.assertContains(response, 'form') + + def test_missing_campaign_invalid(self): + data = self.minimal_valid_data() + del data['campaign'] + response = self.client.post(self.submit_url(), data=data) + self.assertEqual(response.status_code, 200) # form re-rendered, not redirected + self.assertEqual(CampaignRun.objects.count(), 0) + self.assertFormError(response.context['form'], 'campaign', 'This field is required.') + + def test_missing_contact_person_invalid(self): + data = self.minimal_valid_data() + del data['contact_person'] + response = self.client.post(self.submit_url(), data=data) + self.assertEqual(response.status_code, 200) + self.assertEqual(CampaignRun.objects.count(), 0) + self.assertFormError(response.context['form'], 'contact_person', 'This field is required.') + + def test_missing_contact_email_invalid(self): + data = self.minimal_valid_data() + del data['contact_email'] + response = self.client.post(self.submit_url(), data=data) + self.assertEqual(response.status_code, 200) + self.assertEqual(CampaignRun.objects.count(), 0) + self.assertFormError(response.context['form'], 'contact_email', 'This field is required.') + + def test_duplicate_natural_key_submission_shows_friendly_form_error(self): + """Pitfall 4: same campaign+telescope_instrument+window_start(==window_end) collides + on the resolved-window UniqueConstraint -- a friendly non_field_errors banner, never + a 500. + """ + data = self.minimal_valid_data( + telescope_instrument=TELESCOPE_INSTRUMENT, + obs_date=OBS_DATE.isoformat(), + ) + first = self.client.post(self.submit_url(), data=data) + self.assertRedirects(first, self.thanks_url()) + self.assertEqual(CampaignRun.objects.count(), 1) + + second = self.client.post(self.submit_url(), data=data) + self.assertEqual(second.status_code, 200) + self.assertEqual(CampaignRun.objects.count(), 1) # unchanged, no second row + self.assertTrue(second.context['form'].non_field_errors()) + + +class TestCampaignSubmissionObsDateWindow(CampaignSubmissionTestBase): + """260714-ilz: end-to-end POST coverage for the flexible obs_date/window intake + (requirements 1/2/5/7) -- proves no 500, correct DB effect, and correct HTTP status. + """ + + def test_multi_night_range_creates_one_run_with_resolved_window(self): + response = self.client.post( + self.submit_url(), + data=self.minimal_valid_data(obs_date='2027-04-20 -- 2027-05-11'), + ) + self.assertRedirects(response, self.thanks_url()) + self.assertEqual(CampaignRun.objects.count(), 1) + run = CampaignRun.objects.get() + self.assertEqual(run.window_start, date(2027, 4, 20)) + self.assertEqual(run.window_end, date(2027, 5, 11)) + + def test_blank_obs_date_creates_one_tbd_run(self): + response = self.client.post(self.submit_url(), data=self.minimal_valid_data()) + self.assertRedirects(response, self.thanks_url()) + self.assertEqual(CampaignRun.objects.count(), 1) + run = CampaignRun.objects.get() + self.assertIsNone(run.window_start) + self.assertIsNone(run.window_end) + + def test_unparseable_obs_date_re_renders_form_creates_no_run(self): + response = self.client.post( + self.submit_url(), + data=self.minimal_valid_data(obs_date='sometime next spring'), + ) + self.assertEqual(response.status_code, 200) # re-rendered, not a redirect or a 500 + self.assertEqual(CampaignRun.objects.count(), 0) + self.assertIn('obs_date', response.context['form'].errors) + + def test_duplicate_range_submission_shows_friendly_form_error(self): + """Requirement 7: the existing except-IntegrityError handler covers the range case.""" + data = self.minimal_valid_data( + telescope_instrument=TELESCOPE_INSTRUMENT, + obs_date='2027-04-20 -- 2027-05-11', + ) + first = self.client.post(self.submit_url(), data=data) + self.assertRedirects(first, self.thanks_url()) + self.assertEqual(CampaignRun.objects.count(), 1) + + second = self.client.post(self.submit_url(), data=data) + self.assertEqual(second.status_code, 200) # re-rendered, not a 500 + self.assertEqual(CampaignRun.objects.count(), 1) # unchanged, no second row + self.assertTrue(second.context['form'].non_field_errors()) + + +class TestHoneypot(CampaignSubmissionTestBase): + """SUBMIT-04: a tripped honeypot creates nothing, emails nothing, and redirects identically + to a genuine submission. + """ + + def test_honeypot_filled_creates_no_run_and_sends_no_email(self): + data = self.minimal_valid_data(alt_contact_info='I am a bot') + response = self.client.post(self.submit_url(), data=data) + self.assertEqual(CampaignRun.objects.count(), 0) + self.assertEqual(len(mail.outbox), 0) + self.assertRedirects(response, self.thanks_url()) + + def test_honeypot_response_matches_genuine_submission_redirect(self): + genuine_response = self.client.post(self.submit_url(), data=self.minimal_valid_data()) + honeypot_response = self.client.post(self.submit_url(), data=self.minimal_valid_data(alt_contact_info='trap')) + self.assertEqual(genuine_response.url, honeypot_response.url) + self.assertEqual(genuine_response.status_code, honeypot_response.status_code) + + +class TestStaffNotification(CampaignSubmissionTestBase): + """SUBMIT-05: genuine submissions email every is_staff+email user; no PII in the message.""" + + def test_genuine_submission_emails_every_staff_user_with_email(self): + self.client.post(self.submit_url(), data=self.minimal_valid_data()) + self.assertEqual(len(mail.outbox), 1) + self.assertEqual(mail.outbox[0].to, [self.staff_with_email.email]) + + def test_staff_with_blank_email_not_a_recipient(self): + self.client.post(self.submit_url(), data=self.minimal_valid_data()) + self.assertEqual(len(mail.outbox), 1) + self.assertNotIn(self.staff_blank_email.email, mail.outbox[0].to) + + def test_non_staff_user_not_a_recipient(self): + self.client.post(self.submit_url(), data=self.minimal_valid_data()) + self.assertEqual(len(mail.outbox), 1) + self.assertNotIn(self.non_staff_user.email, mail.outbox[0].to) + + def test_email_contains_no_pii(self): + """D-04: subject/body must never contain contact_person, contact_email, + telescope_instrument, or campaign name. + """ + self.client.post( + self.submit_url(), + data=self.minimal_valid_data(telescope_instrument=TELESCOPE_INSTRUMENT), + ) + self.assertEqual(len(mail.outbox), 1) + sent = mail.outbox[0] + for pii in (CONTACT_PERSON, CONTACT_EMAIL, TELESCOPE_INSTRUMENT, self.campaign.name): + self.assertNotIn(pii, sent.subject) + self.assertNotIn(pii, sent.body) + + def test_no_staff_with_email_sends_no_email(self): + self.staff_with_email.email = '' + self.staff_with_email.save() + self.client.post(self.submit_url(), data=self.minimal_valid_data()) + self.assertEqual(len(mail.outbox), 0) + + +class TestSubmissionFormSiteSearchWidget(CampaignSubmissionTestBase): + """D-09/D-10/22-REVIEWS.md findings 1 and 7: the public form's site_raw field is a + live-search widget wired to campaigns:site_search, with NO create-new-site escape + hatch (that stays staff-only on the approval queue -- Task 2). + """ + + def test_form_renders_hx_get_and_corrected_trigger_grammar(self): + response = self.client.get(self.submit_url()) + self.assertContains(response, 'hx-get') + self.assertContains(response, reverse('campaigns:site_search')) + # Django HTML-escapes widget attribute values (`>` -> `>`, `"` -> `"`), so + # assert on escaping-immune substrings either side of the event-filter bracket. + self.assertContains(response, 'hx-trigger="input[this.value.length') + self.assertContains(response, '] changed delay:300ms"') + # 22-REVIEWS.md finding 1: the malformed filter-after-delay ordering must never + # regress back in. + self.assertNotContains(response, 'delay:300ms[') + + def test_form_renders_suggestions_container(self): + response = self.client.get(self.submit_url()) + self.assertContains(response, '
25 so pagination (D-11) is genuinely exercised +_BASE_DATE = date(2026, 6, 1) + +CONTACT_PERSON = 'Jane Coordinator' +CONTACT_EMAIL = 'jane@example.org' + + +class CampaignViewTestBase(TestCase): + """Shared fixture: one campaign with 30 CampaignRun rows, one empty campaign, one staff user.""" + + @classmethod + def setUpTestData(cls) -> None: + cls.campaign = TargetList.objects.create(name='3I/ATLAS') + cls.empty_campaign = TargetList.objects.create(name='Empty Campaign') + cls.staff_user = User.objects.create_user(username='staffcoordinator', password='pw', is_staff=True) + + cls.runs = [] + for i in range(_TOTAL_RUNS): + window_date = _BASE_DATE + timedelta(days=i) + kwargs = { + 'campaign': cls.campaign, + 'telescope_instrument': f'FTN/MuSCAT3-{i}', + 'window_start': window_date, + 'window_end': window_date, + } + if i == _TOTAL_RUNS - 1: + # Most-recent row (highest window_start -- always page 1, first row per D-10). + # Carries the seeded contact PII and open_to_collaboration=True so VIEW-03/ + # VIEW-04 assertions never depend on which pagination page a row lands on. + kwargs.update( + run_status=CampaignRun.RunStatus.PLANNED, + approval_status=CampaignRun.ApprovalStatus.APPROVED, + contact_person=CONTACT_PERSON, + contact_email=CONTACT_EMAIL, + open_to_collaboration=True, + ) + elif i == _TOTAL_RUNS - 2: + kwargs.update( + run_status=CampaignRun.RunStatus.OBSERVED, + approval_status=CampaignRun.ApprovalStatus.PENDING_REVIEW, + ) + elif i == _TOTAL_RUNS - 3: + kwargs.update( + run_status=CampaignRun.RunStatus.CANCELLED, + approval_status=CampaignRun.ApprovalStatus.REJECTED, + ) + else: + kwargs.update( + run_status=_CYCLE_RUN_STATUSES[i % len(_CYCLE_RUN_STATUSES)], + approval_status=_CYCLE_APPROVAL_STATUSES[i % len(_CYCLE_APPROVAL_STATUSES)], + ) + cls.runs.append(CampaignRun.objects.create(**kwargs)) + + cls.most_recent_run = cls.runs[-1] + + def table_url(self, campaign=None): + return reverse('campaigns:table', kwargs={'pk': (campaign or self.campaign).pk}) + + def list_url(self): + return reverse('campaigns:list') + + @staticmethod + def _row_value(record, field): + """Read a field from a table row's record -- a dict for non-staff (.values()) rows, + a model instance for staff rows (RESEARCH.md Pitfall 2 dict-vs-model-instance).""" + if isinstance(record, dict): + return record[field] + return getattr(record, field) + + +class TestCampaignRunTableView(CampaignViewTestBase): + """VIEW-01: table lists all runs for a campaign, 25/page, default-sorted window_start desc. + + These assertions are about generic table mechanics (pagination, sort, full row-status + coverage), not approval-status visibility gating -- exercised via the staff client so + D-09's non-staff `.exclude(approval_status=PENDING_REVIEW)` (added in Plan 04) doesn't + change the expected row counts here. D-09 visibility itself is covered separately by + `TestNonStaffPendingReviewHidden`. + """ + + def test_anonymous_get_returns_200(self): + response = self.client.get(self.table_url()) + self.assertEqual(response.status_code, 200) + + def test_first_page_shows_25_rows_and_second_page_exists(self): + self.client.force_login(self.staff_user) + response = self.client.get(self.table_url()) + table = response.context['table'] + self.assertEqual(len(table.page.object_list), 25) + self.assertGreaterEqual(table.paginator.num_pages, 2) + + def test_default_load_shows_every_seeded_run_status_value(self): + self.client.force_login(self.staff_user) + response = self.client.get(self.table_url()) + table = response.context['table'] + seen_statuses = {self._row_value(row.record, 'run_status') for row in table.page.object_list} + self.assertEqual(seen_statuses, set(CampaignRun.RunStatus.values)) + + def test_default_sort_is_window_start_desc_tbd_last(self): + """D-04: resolved rows lead (most recent window_start first); a TBD row (both + window fields null) sorts last -- portably across backends via + F('window_start').desc(nulls_last=True), never relying on the DB's own implicit + NULL-ordering default (SQLite/PostgreSQL disagree on that direction).""" + CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='TBD-Telescope', + contact_person='TBD Coordinator', + approval_status=CampaignRun.ApprovalStatus.APPROVED, + ) + response = self.client.get(self.table_url()) + table = response.context['table'] + first_record = table.page.object_list[0].record + self.assertEqual(self._row_value(first_record, 'window_start'), self.most_recent_run.window_start) + + last_page_rows = list(table.paginator.page(table.paginator.num_pages).object_list) + last_record = last_page_rows[-1].record + self.assertIsNone(self._row_value(last_record, 'window_start')) + + +class TestWindowColumnRendering(TestCase): + """D-03/D-05: TBD badge, single-date, and range-arrow rendering for the window column. + + Exercises CampaignRunTable.render_window_start() directly (no HTTP round trip needed -- + this is purely about the render method's output, mirroring test_campaign_approval.py's + TestApprovalQueueSiteVisibility precedent for render_site()). + """ + + @classmethod + def setUpTestData(cls) -> None: + cls.campaign = TargetList.objects.create(name='Render Campaign') + + def test_tbd_row_renders_tbd_indicator(self): + run = CampaignRun.objects.create( + campaign=self.campaign, telescope_instrument='TBD Scope', contact_person='Render Contact' + ) + cell = CampaignRunTable([run]).rows[0].get_cell('window_start') + self.assertIn('TBD', cell) + + def test_tbd_row_with_raw_text_renders_tooltip(self): + """D-08: a TBD row with original_obs_date_raw set carries a title tooltip.""" + run = CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='TBD Scope', + contact_person='Render Contact Raw', + original_obs_date_raw='TBD pending Cycle 2', + ) + cell = CampaignRunTable([run]).rows[0].get_cell('window_start') + self.assertIn('title="TBD pending Cycle 2"', cell) + + def test_tbd_row_with_blank_raw_text_renders_no_title(self): + """D-08: a blank original_obs_date_raw renders the plain TBD badge, no title attribute.""" + run = CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='TBD Scope', + contact_person='Render Contact Blank', + ) + cell = CampaignRunTable([run]).rows[0].get_cell('window_start') + self.assertNotIn('title=', cell) + + def test_tbd_row_with_markup_raw_text_is_escaped(self): + """T-20-03: angle-bracket markup in original_obs_date_raw is HTML-escaped, not rendered live.""" + run = CampaignRun.objects.create( + campaign=self.campaign, + telescope_instrument='TBD Scope', + contact_person='Render Contact Markup', + original_obs_date_raw='', + ) + cell = CampaignRunTable([run]).rows[0].get_cell('window_start') + self.assertNotIn(' +
diff --git a/src/templatetags/solsys_code_extras.py b/src/templatetags/solsys_code_extras.py index 22bb82df..fff0161d 100644 --- a/src/templatetags/solsys_code_extras.py +++ b/src/templatetags/solsys_code_extras.py @@ -1,4 +1,5 @@ from django import template +from tom_targets.models import TargetList register = template.Library() @@ -11,3 +12,27 @@ def ephem_button(context): context = {'button_text': 'Ephemeris'} return context + + +@register.inclusion_tag('solsys_code/partials/campaign_links.html', takes_context=True) +def campaign_links(context): + """ + Returns every campaign (TargetList with >= 1 CampaignRun) the rendered target belongs to, + discovered via TargetList membership -- never via the run's optional target FK (D-01). + """ + target = context.get('target') + campaigns = ( + TargetList.objects.filter(targets=target, campaign_runs__isnull=False).distinct() + if target + else TargetList.objects.none() + ) + return {'campaigns': campaigns} + + +@register.inclusion_tag('solsys_code/partials/campaigns_nav_link.html', takes_context=True) +def campaigns_nav_link(context): + """ + Static navbar entry linking to the campaigns list page (D-03). No per-request data needed; + the partial reads `request` from the surrounding page context for the active-nav check. + """ + return {} From 4410c32cdf0bd3d4f0709f155231e9232835ca74 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Sun, 19 Jul 2026 00:31:10 -0700 Subject: [PATCH 2/3] feat: Phase 24-25 runbook docs, backfill command, and PR-review fixes Folds in code/template/docs changes made on issue37-telescope-runs-calendar since the original squash point (47dcfb9): the operator/usage runbook (docs/installation.rst, docs/notebooks.rst, docs/runbooks/), the backfill_range_calendar_events management command (Phase 25's fix for range-window CalendarEvent projection), and the PR-review fixes from quick task 260718-dih (guarded calendar-sync loop in _set_run_status, fail-fast cross-month rejection, anchored partial-night token matching). Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 4 + docs/index.rst | 1 + docs/installation.rst | 33 +++ docs/notebooks.rst | 11 + docs/runbooks/telescope_runs_calendar.rst | 249 ++++++++++++++++++ pyproject.toml | 1 + solsys_code/campaign_views.py | 140 +++++++--- .../backfill_range_calendar_events.py | 103 ++++++++ .../commands/load_telescope_runs.py | 11 +- solsys_code/telescope_runs.py | 51 ++-- .../test_backfill_range_calendar_events.py | 124 +++++++++ solsys_code/tests/test_campaign_approval.py | 209 ++++++++++++--- solsys_code/tests/test_load_telescope_runs.py | 20 ++ solsys_code/tests/test_telescope_runs.py | 20 +- 14 files changed, 865 insertions(+), 112 deletions(-) create mode 100644 docs/runbooks/telescope_runs_calendar.rst create mode 100644 solsys_code/management/commands/backfill_range_calendar_events.py create mode 100644 solsys_code/tests/test_backfill_range_calendar_events.py diff --git a/.gitignore b/.gitignore index cdcd7a34..8ab2d465 100644 --- a/.gitignore +++ b/.gitignore @@ -166,3 +166,7 @@ src/data/ # Claude Code / GSD tooling (local install + machine-specific config) .claude/ + +# graphify skill — generated knowledge-graph cache/build output +.planning/graphs/ +graphify-out/ diff --git a/docs/index.rst b/docs/index.rst index febffefe..38a3a451 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,5 +21,6 @@ FOMO, which stands for Follow-up Observations of Moving Objects is a Target and Home page Installation and Getting Started Design + Runbooks API Reference Notebooks diff --git a/docs/installation.rst b/docs/installation.rst index 9f78699a..ad52ee68 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -115,3 +115,36 @@ You can import new Targets into FOMO by clicking on Targets->Targets in the menu In addition sidereal targets can be imported from Simbad and TNS. FOMO will then fetch the details from the selected service and display the details for review and user modification (if desired). Otherwise you can hit the `Submit` button to create the Target. You will then be redirected to the Target detail page. + +.. _running-management-commands: + +Running FOMO Management Commands +-------------------------------------- + +Beyond the ``migrate`` and ``createsuperuser`` commands above, FOMO ships a +number of custom management commands (for example, syncing telescope +schedules onto the shared calendar, or importing a campaign coordination +CSV). All of them are invoked the same way: with the project's virtual +environment (or conda/mamba environment) activated, run +``python3 manage.py `` from the repository root. + +.. code-block:: console + + >> source ~/path-to-new-venv/bin/activate + >> python3 manage.py [options] + +Every management command supports ``--help``, which prints its usage, its +positional arguments, and any optional flags: + +.. code-block:: console + + >> python3 manage.py --help + +If a command's underlying data model has changed (for example, after +upgrading FOMO to a newer release), you may need to re-apply +``python3 manage.py migrate`` -- see "Initializing FOMO and the database" +above -- before running the command again. + +For a task-oriented walkthrough of the specific commands and staff actions +that keep the telescope runs calendar and campaign coordination up to date, +see the Telescope Runs Calendar Operator Runbook. diff --git a/docs/notebooks.rst b/docs/notebooks.rst index 7f7e544d..9defe996 100644 --- a/docs/notebooks.rst +++ b/docs/notebooks.rst @@ -4,3 +4,14 @@ Notebooks .. toctree:: Introducing Jupyter Notebooks + +Demonstration Notebooks +------------------------ + +.. toctree:: + + Telescope Runs (site / ephemeris helper) + Loading Telescope Runs + Syncing the LCO Observation Calendar + Syncing the Gemini Observation Calendar + Importing a Campaign CSV diff --git a/docs/runbooks/telescope_runs_calendar.rst b/docs/runbooks/telescope_runs_calendar.rst new file mode 100644 index 00000000..6e14ee02 --- /dev/null +++ b/docs/runbooks/telescope_runs_calendar.rst @@ -0,0 +1,249 @@ +Telescope Runs Calendar — Operator Runbook +=========================================== + +This is the how-to-run companion to the +:doc:`/design/telescope_runs_calendar` design document -- see that page for +the *why* (dip-corrected sunset/sunrise, the -15 deg dark window, the +queue-vs-classical scheduling models, and so on). This page is deliberately +task-oriented: it walks through each management command and staff action as +a "How do I...?" question, followed by a quick-reference cheat-sheet and a +troubleshooting section. + +This runbook assumes you already have FOMO installed and can run +``python3 manage.py `` from an activated virtual environment; see +:ref:`running-management-commands` if you need that background first. + +How do I load a classical telescope schedule? +----------------------------------------------- + +``load_telescope_runs`` reads a plain-text schedule file -- one classical +run per line, e.g. ``NTT EFOSC2 allocation 9-13 July`` -- and expands each +run into one ``CalendarEvent`` per observing night, with sunset/sunrise and +the -15 deg dark window computed for that night's site. Running it again on +an unchanged file is a no-op; running it after the file changes creates or +updates only the affected nights. + +.. code-block:: console + + >> python3 manage.py load_telescope_runs path/to/schedule.txt + +How do I sync LCO/SOAR queue observations? +--------------------------------------------- + +``sync_lco_observation_calendar`` syncs LCO and SOAR queue +``ObservationRecord`` rows onto the calendar as one ``CalendarEvent`` per +record, keyed on the LCO portal URL. A record still awaiting placement by +the LCO scheduler becomes a ``[QUEUED]`` scheduling-window banner; once the +scheduler places it, re-running the command updates the same event in +place to the real placed block times. + +The required ``--proposal`` flag accepts: + +* a single proposal code, e.g. ``--proposal LCO2026A-001``; +* a comma-separated list of codes, e.g. ``--proposal A,B,C`` (matches only + those exact codes -- no substring leakage, so ``--proposal A`` never also + matches a proposal literally named ``AB``); +* the case-insensitive token ``ALL``, which syncs every LCO and SOAR record + regardless of proposal. + +.. code-block:: console + + >> python3 manage.py sync_lco_observation_calendar --proposal LCO2026A-001 + >> python3 manage.py sync_lco_observation_calendar --proposal ALL + +How do I sync Gemini queue observations? +------------------------------------------- + +``sync_gemini_observation_calendar`` syncs every submitted Gemini +Target-of-Opportunity ``ObservationRecord`` (``facility='GEM'``) onto the +calendar, unconditionally. + +.. code-block:: console + + >> python3 manage.py sync_gemini_observation_calendar + +Unlike the LCO/SOAR sync above, this command has **no proposal or filter +flag at all** -- it always processes every Gemini ``ObservationRecord`` in +the database. If you're used to the ``--proposal`` flag from the LCO +section, do not expect an equivalent here; there is nothing to pass. Each +record's observing window comes from its explicit +``windowDate``/``windowTime``/``windowDuration`` parameters when present, +or is otherwise derived from its Target-of-Opportunity type (a Rapid ToO +gets a 24-hour window from submission; a Standard ToO gets a 24-hour to +7-day window). + +How do I mark a run cancelled or weathered-out? +-------------------------------------------------- + +Once a campaign run is approved, the approval queue's **Decided** table +shows "Mark Cancelled" (``action=mark_cancelled``) and "Mark Weathered" +(``action=mark_weather_failure``) buttons on that row's Actions column +(they appear for any approved run regardless of its current observing +status). Clicking one immediately and publicly prepends +``[CANCELLED]`` or ``[WEATHERED]`` to the title of **every** +``CalendarEvent`` associated with that run -- including every per-night +event of a multi-night range-window run -- on the shared campaign calendar +that anonymous visitors can see. There is no separate confirmation step and +no revert button, but the action is a safe, idempotent no-op to re-click: +clicking the same button again, or clicking the other button to correct a +mis-click, simply re-applies the new prefix without creating duplicate +events or losing any data. + +How do I bootstrap-import a campaign from a CSV? +---------------------------------------------------- + +``import_campaign_csv`` bulk-imports a campaign coordination spreadsheet +(for example, a community campaign's shared observing-run tracking sheet) +into ``CampaignRun`` rows, one row per CSV line. + +.. code-block:: console + + >> python3 manage.py import_campaign_csv --campaign "3I/ATLAS" path/to/campaign.csv + +.. note:: + **Re-import gotcha:** re-running this command over the same + ``--campaign`` always resets every row's ``target`` field back to its + auto-resolved value. If a staff member manually corrected a row's + ``target`` in the Django admin after a previous import, that correction + is silently overwritten the next time this command runs over the same + campaign CSV. This is expected behavior for a bootstrap-import command, + not a bug -- but it is easy to be surprised by, so re-import + deliberately, not routinely. + +How do I backfill calendar events for older approved range-window runs? +---------------------------------------------------------------------------- + +``backfill_range_calendar_events`` is a one-off command for a narrow +historical gap: a multi-night range-window ``CampaignRun`` that was already +approved and site-resolved *before* per-night calendar projection existed +never got any ``CalendarEvent`` at all, and normal approval/resolve actions +only project events going forward, not retroactively. This command finds +every already-approved, site-resolved range-window run with no existing +calendar event and projects one per night, exactly as if it had just been +approved. + +Always run with ``--dry-run`` first to see which runs would be backfilled, +with no database writes: + +.. code-block:: console + + >> python3 manage.py backfill_range_calendar_events --dry-run + >> python3 manage.py backfill_range_calendar_events + +The command is safe to re-run: a run that already has a calendar event is +skipped, so running it again after a real backfill is a no-op. + +.. _command-cheat-sheet: + +Command cheat-sheet +----------------------- + +.. list-table:: + :header-rows: 1 + :widths: 30 30 40 + + * - Command + - Key flags + - One-line description + * - ``load_telescope_runs`` + - ```` (positional) + - Ingest a classical-schedule text file into per-night CalendarEvents. + * - ``sync_lco_observation_calendar`` + - ``--proposal `` (required) + - Sync LCO/SOAR queue ObservationRecords to CalendarEvents. + * - ``sync_gemini_observation_calendar`` + - (none) + - Sync every Gemini ToO ObservationRecord to CalendarEvents. + * - ``import_campaign_csv`` + - ``--campaign `` (required), ```` (positional) + - Bootstrap-import a campaign coordination CSV into CampaignRun rows. + * - ``backfill_range_calendar_events`` + - ``--dry-run`` (optional) + - One-off backfill of CalendarEvents for older approved range-window runs. + +Troubleshooting +------------------ + +These are failure modes that have actually been observed running these +commands against real data -- not a speculative list of every possible +exception. Every example below uses synthetic placeholder names, emails, +and telescope/instrument strings; no real contact information appears +anywhere on this page. + +Observatory missing timezone +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Any command that needs to compute sunset/sunrise or the -15 deg dark +window for a site (``sync_lco_observation_calendar``, +``backfill_range_calendar_events``, and any future projection over that +Observatory) will fail with an error like this, observed running a real +backfill against the dev database: + +.. code-block:: console + + Observatory 'FTN' (obscode=F65) has no timezone set + +**Fix:** the ``Observatory`` record for that site is missing its +``timezone`` field. Set it to a valid IANA timezone name -- for example +``"America/Santiago"`` -- via the Django admin, or via the +``CreateObservatory`` form, then re-run the sync/backfill command for that +site. Until the field is set, every projection or backfill attempt against +that ``Observatory`` record will keep failing with the same error; it is +not a one-time fluke. + +Per-line / per-record skip-and-log behaviour +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Each ingest/sync command follows the same shared invariant: **one bad row +never aborts the whole run.** A problem with a single line or record is +logged and skipped, and the command continues to the end, reporting a +summary count. + +* ``load_telescope_runs`` skips and logs any schedule line it cannot parse, + or whose telescope name doesn't resolve to a known ``Observatory`` + (a caught ``ValueError``/``Observatory.DoesNotExist``), and reports a + ``skipped: N`` count in its final summary line, e.g.:: + + Line 12: Observatory 'XYZ' (obscode=???) has no timezone set (line text: 'XYZ Instrument 1-5 July') + Done. lines processed: 20, created: 95, updated: 0, unchanged: 0, skipped: 1 + +* ``sync_lco_observation_calendar`` falls back to a coarse, clearly-labelled + ``[UNVERIFIED]`` telescope name (instead of skipping the record) when its + per-record live telescope-label API call times out or returns an + unmapped site/telescope code. This is tracked as its own + ``telescope_api_failed`` counter, separate from ``skipped``, and the + record still gets a ``CalendarEvent``. + +* ``backfill_range_calendar_events`` skips a candidate run on a + ``ValueError`` (for example, the Observatory-timezone gap above) and + continues to the next candidate, never aborting the whole backfill. + +``import_campaign_csv`` unresolved rows +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A row whose ``Site Code`` cell doesn't resolve to a known ``Observatory``, +or whose ``Obs. Date`` cell doesn't parse into a concrete window, is never +silently dropped. Instead, the row still imports (or updates) as a +``CampaignRun``, flagged with ``site_needs_review`` and/or +``window_needs_review``, and both counts appear in the command's final +summary line, e.g.:: + + Done. created: 12, updated: 3, unchanged: 40, skipped: 1, site_needs_review: 2, window_needs_review: 1 + +Rows flagged ``site_needs_review`` surface in the approval queue's "Sites +Needing Review" card so staff can resolve them without re-running the +import. + +Also recall the re-import ``target``-reset gotcha covered above under "How +do I bootstrap-import a campaign from a CSV?": re-running +``import_campaign_csv`` over the same ``--campaign`` always resets every +row's ``target`` back to its auto-resolved value, silently overwriting any +manual correction made since the previous import. + +See also +----------- + +* The :ref:`command cheat-sheet ` above for exact flag + syntax. +* :doc:`/design/telescope_runs_calendar` for the astronomy and data-model + rationale behind these commands. diff --git a/pyproject.toml b/pyproject.toml index 5990a181..7f24188a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dev = [ "pytest", "pytest-cov", # Used to report total code coverage "ruff", # Used for static linting of files + "graphifyy", # (not a typo) queryable knowledge graph builder for AI tools ] [build-system] diff --git a/solsys_code/campaign_views.py b/solsys_code/campaign_views.py index 62217023..ee300a66 100644 --- a/solsys_code/campaign_views.py +++ b/solsys_code/campaign_views.py @@ -13,7 +13,7 @@ import logging import re -from datetime import date, datetime +from datetime import date, datetime, timedelta from datetime import time as dt_time from datetime import timezone as dt_timezone @@ -21,7 +21,7 @@ from django.contrib.auth.models import User from django.core.mail import send_mail from django.db import IntegrityError, transaction -from django.db.models import Case, CharField, Count, EmailField, F, Value, When +from django.db.models import Case, CharField, Count, EmailField, F, Q, Value, When from django.http import HttpResponse, HttpResponseBadRequest from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse, reverse_lazy @@ -389,11 +389,23 @@ def get_context_data(self, **kwargs): } +def _calendar_event_title(run: CampaignRun) -> str: + """Builds a CampaignRun's calendar-event title, with a D-06 window-context suffix for a + range window. Single source of truth for the title -- reused by both + ``_project_calendar_event()`` (creation) and ``_set_run_status()`` (status update) so the + suffix can never drift out of sync between the two call sites (Pitfall 1). + """ + base = f'{run.campaign.name}: {run.telescope_instrument}' + if run.window_start != run.window_end: + return f'{base} (window {run.window_start}..{run.window_end})' + return base + + def _project_calendar_event(run: CampaignRun) -> bool: """CAL-01/CAL-02 CalendarEvent projection (D-08), extracted from the approve branch. Returns True when ``insert_or_create_calendar_event()`` was actually called (an event was - created/updated), False when projection was skipped by design (range/TBD run, or missing + created/updated), False when projection was skipped by design (TBD run, or missing telescope_instrument/site) -- 22-REVIEWS.md finding 6: this bool drives the resolve_site action's two distinct success messages. RAISES ValueError when ``sun_event()`` fails (e.g. a Tier-2-resolved site with a blank ``timezone`` -- CR-01), and MAY RAISE on any other @@ -404,22 +416,29 @@ def _project_calendar_event(run: CampaignRun) -> bool: ``approve()`` has no retry surface to protect and instead catches-and-swallows the ValueError case specifically at its call site to preserve its original behavior (approval still succeeds even when the calendar entry couldn't be projected). + + A ground range-window run now projects one dip-corrected event per night (D-02); a + single-night run keeps its existing bare-key single event. Only TBD runs (window_start + is None), unresolved-site runs, and missing-telescope_instrument runs are excluded from + projection. """ - # D-06/CAL-01: CalendarEvent.start_time/end_time are non-nullable -- only project a - # single concrete night (window_start == window_end); a resolved site is required to - # pick the ground-vs-space branch. A range, TBD run, or unresolved site simply doesn't - # get a CalendarEvent yet. - if not (run.telescope_instrument and run.site and run.window_start and run.window_start == run.window_end): + # D-01/CAL-01: CalendarEvent.start_time/end_time are non-nullable -- a concrete window + # (both window_start and window_end set) and a resolved site are required to pick the + # ground-vs-space branch. A TBD run, unresolved site, or missing telescope_instrument + # simply doesn't get a CalendarEvent yet. + if not (run.telescope_instrument and run.site and run.window_start and run.window_end): return False event_fields = { - 'title': f'{run.campaign.name}: {run.telescope_instrument}', + 'title': _calendar_event_title(run), 'description': run.observation_details, 'target_list': run.campaign, # CAL-02 'telescope': run.telescope_instrument, } if run.site.observations_type == Observatory.SATELLITE_OBSTYPE: # Space-based observatory: no fixed horizon for sun_event() to work against -- use - # a midnight-UTC placeholder spanning the window date. + # a midnight-UTC placeholder spanning the window date. D-05: this branch's date-math + # is unchanged by the ground-branch's per-night rewrite below -- a satellite range + # still yields exactly one whole-day-span event under the bare key. event_fields['start_time'] = datetime.combine(run.window_start, dt_time(0, 0), tzinfo=dt_timezone.utc) event_fields['end_time'] = datetime.combine(run.window_end, dt_time(23, 59), tzinfo=dt_timezone.utc) # Never construct CalendarEvent directly -- always route through the shared helper @@ -439,19 +458,32 @@ def _project_calendar_event(run: CampaignRun) -> bool: # deliberate simplification for this milestone; scope this to Observatory.OPTICAL_OBSTYPE # explicitly, with OCCULTATION/RADAR falling back to no projection, when those site types # get real support. - try: - sunset, sunrise = sun_event(run.site, run.window_start, kind='sun') - except ValueError: - logger.debug( - 'sun_event(sun) raised for site=%s date=%s; re-raising so callers that need the ' - 'retry guarantee (resolve_site) see this as a failure, not a by-design skip.', - run.site, - run.window_start, - ) - raise # CR-01: never silently swallow this -- see docstring above. - event_fields['start_time'] = sunset.to_datetime(timezone=dt_timezone.utc).replace(microsecond=0) - event_fields['end_time'] = sunrise.to_datetime(timezone=dt_timezone.utc).replace(microsecond=0) - insert_or_create_calendar_event({'url': f'CAMPAIGN:{run.pk}'}, fields=event_fields) + # + # D-02/D-03: mirrors load_telescope_runs' E - S + 1 inclusive-range idiom. A single-night + # run (n_nights == 1) reproduces today's exact bare-CAMPAIGN:{pk}-keyed single event; a + # range run (n_nights > 1) creates one CAMPAIGN:{pk}:{date.isoformat()}-keyed event per + # night. A mid-loop sun_event() ValueError (CR-01) re-raises immediately, leaving any + # already-created earlier nights' events in place -- accepted partial projection, no + # transaction.atomic() wrap (RESEARCH Assumption A3). + n_nights = (run.window_end - run.window_start).days + 1 + is_range = n_nights > 1 + for i in range(n_nights): + night = run.window_start + timedelta(days=i) + try: + sunset, sunrise = sun_event(run.site, night, kind='sun') + except ValueError: + logger.debug( + 'sun_event(sun) raised for site=%s date=%s; re-raising so callers that need the ' + 'retry guarantee (resolve_site) see this as a failure, not a by-design skip.', + run.site, + night, + ) + raise # CR-01: never silently swallow this -- see docstring above. + night_fields = dict(event_fields) + night_fields['start_time'] = sunset.to_datetime(timezone=dt_timezone.utc).replace(microsecond=0) + night_fields['end_time'] = sunrise.to_datetime(timezone=dt_timezone.utc).replace(microsecond=0) + url = f'CAMPAIGN:{run.pk}' if not is_range else f'CAMPAIGN:{run.pk}:{night.isoformat()}' + insert_or_create_calendar_event({'url': url}, fields=night_fields) return True @@ -707,7 +739,7 @@ def _resolve_site(self, request, pk): def _set_run_status(self, request, pk, action): """D-03/D-04/D-05: mark an already-APPROVED run cancelled or weathered, and update - its linked CAMPAIGN:{pk} CalendarEvent in place if (and only if) one already exists. + EVERY CalendarEvent belonging to this run in place, if (and only if) any already exist. Mirrors ``_resolve_site()``'s shape: a server-side business-logic guard (never trust the Decided-table button was only rendered for an APPROVED row -- T-23-01), then a @@ -717,12 +749,19 @@ def _set_run_status(self, request, pk, action): and the write must never reach ``refresh_from_db()`` (which would raise ``CampaignRun.DoesNotExist`` on a deleted row) or silently report false success. - A run whose window/site never projected a CAMPAIGN:{pk} event (a range/TBD run, or - one with an unresolved site) still gets its run_status set, but is never handed to + A run whose window/site never projected any CalendarEvent (a TBD run, or one with an + unresolved site) still gets its run_status set, but is never handed to ``insert_or_create_calendar_event()`` -- that helper's create-path requires non-nullable start_time/end_time this call deliberately omits, and would raise (T-23-06/RESEARCH Pitfall 1). ``_project_calendar_event()`` itself is never called or - modified here. + modified here. A range-window run now has one-or-more per-night events, all of which + get updated here via the combined trailing-colon queryset (D-04). + + The calendar-sync loop below is wrapped in a non-reverting try/except (PR-REVIEW-F1): + ``run_status`` is already committed by the conditional ``.update()`` above by the time + the loop runs, so a sync failure (DB/network/runtime) never reverts it and never + surfaces as an uncaught 500 -- it logs the exception and warns the user that the status + change was saved but the calendar entry needs a retry of the same action. """ run = get_object_or_404(CampaignRun, pk=pk) @@ -746,18 +785,41 @@ def _set_run_status(self, request, pk, action): run.refresh_from_db() - # D-05/T-23-06: only touch the linked CalendarEvent if one already exists -- never - # fabricate one for a run that never had a projected event (range/TBD/unresolved-site - # runs never reach _project_calendar_event()'s single-night+resolved-site branch). - if CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').exists(): - prefix = _RUN_STATUS_CALENDAR_PREFIX[new_run_status] - insert_or_create_calendar_event( - {'url': f'CAMPAIGN:{run.pk}'}, - fields={ - 'title': f'{prefix} {run.campaign.name}: {run.telescope_instrument}', - 'description': f'{run.observation_details}\nRun status: {run.get_run_status_display()}', - }, - ) + # D-04/T-23-06: find and update EVERY CalendarEvent belonging to this run -- the + # legacy bare single-night key AND any new per-night keys. The trailing colon on the + # startswith prefix is required (Pitfall 2): without it, run.pk=3 would also match + # CAMPAIGN:34:... events belonging to a different run. Never fabricate an event for a + # run that never projected one (a TBD run, or one with an unresolved site or missing + # telescope_instrument never reaches _project_calendar_event()'s date-math branch). + matching_events = CalendarEvent.objects.filter( + Q(url=f'CAMPAIGN:{run.pk}') | Q(url__startswith=f'CAMPAIGN:{run.pk}:') + ) + if matching_events.exists(): + # PR-REVIEW-F1: run_status is already committed above -- this loop is wrapped in a + # non-reverting try/except (mirrors _resolve_site()'s projection guard) so a sync + # failure never reverts the status change and never bubbles up as an uncaught 500. + try: + prefix = _RUN_STATUS_CALENDAR_PREFIX[new_run_status] + for event in matching_events: + insert_or_create_calendar_event( + {'url': event.url}, + fields={ + # Pitfall 1: reuse the shared _calendar_event_title() helper -- a + # re-derived inline f-string here would differ from the stored range + # title, get treated as a real change by the no-churn diff, and + # silently strip the D-06 window suffix from every night's event. + 'title': f'{prefix} {_calendar_event_title(run)}', + 'description': f'{run.observation_details}\nRun status: {run.get_run_status_display()}', + }, + ) + except Exception: + logger.exception('Calendar sync failed for CampaignRun %s during _set_run_status.', pk) + messages.warning( + request, + 'Run status was updated, but the calendar entry could not be synced -- ' + 'retry the same action to sync the calendar.', + ) + return redirect('campaigns:approval_queue') messages.success(request, 'Run status updated.') return redirect('campaigns:approval_queue') diff --git a/solsys_code/management/commands/backfill_range_calendar_events.py b/solsys_code/management/commands/backfill_range_calendar_events.py new file mode 100644 index 00000000..e27ce712 --- /dev/null +++ b/solsys_code/management/commands/backfill_range_calendar_events.py @@ -0,0 +1,103 @@ +import logging +from typing import Any + +from django.core.management.base import BaseCommand, CommandParser +from django.db.models import F, Q +from tom_calendar.models import CalendarEvent + +from solsys_code.campaign_views import _project_calendar_event +from solsys_code.models import CampaignRun + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + """One-off backfill: project CalendarEvents for already-APPROVED range-window CampaignRuns. + + Projection only fires on the approve / resolve_site POST actions (never retroactively), so + any range-window run approved before Phase 25's per-night projection existed (e.g. the real + GS-2026A-FT-115 row, CampaignRun pk=34) stays eventless forever without this command + (D-07/FIX-08). Re-runnable and idempotent: a run that already has a CAMPAIGN:{pk}* event is + skipped. + """ + + help = ( + 'One-off backfill: project CalendarEvents for already-APPROVED, site-resolved ' + 'range-window CampaignRuns that were approved before per-night projection existed.' + ) + + def add_arguments(self, parser: CommandParser) -> None: + """Parse command line arguments.""" + parser.add_argument( + '--dry-run', + action='store_true', + help='Report which runs would be backfilled without writing any CalendarEvent rows.', + ) + # No return statement — BaseCommand.add_arguments() returns None + + def handle(self, *args: Any, **options: Any) -> str | None: + """Find qualifying runs and delegate projection to campaign_views._project_calendar_event(). + + Returns: + str | None: None on completion. + """ + dry_run = options['dry_run'] + + # Deliberately not filtered by site.observations_type -- D-07's "ground-based site" + # describes the real motivating data (pk=34), and _project_calendar_event() already + # routes ground vs satellite correctly, so a hypothetical satellite range candidate is + # handled safely too. + candidates = list( + CampaignRun.objects.filter( + approval_status=CampaignRun.ApprovalStatus.APPROVED, + site__isnull=False, + window_start__isnull=False, + ).exclude(window_start=F('window_end')) + ) + + backfilled_count = 0 + skipped_count = 0 + failed_count = 0 + would_backfill_count = 0 + + for run in candidates: + # Trailing colon (Pitfall 2): a bare CAMPAIGN:{pk} substring match would also hit + # CAMPAIGN:{pk}0:... for a longer pk -- combine bare-key and per-night-key lookups. + already = CalendarEvent.objects.filter( + Q(url=f'CAMPAIGN:{run.pk}') | Q(url__startswith=f'CAMPAIGN:{run.pk}:') + ).exists() + if already: + skipped_count += 1 + continue + + if dry_run: + self.stdout.write( + f'Would backfill run pk={run.pk} ({run.campaign.name}: {run.telescope_instrument}) ' + f'window {run.window_start}..{run.window_end}' + ) + would_backfill_count += 1 + continue + + try: + _project_calendar_event(run) + backfilled_count += 1 + except ValueError as exc: + logger.debug('_project_calendar_event() raised for run pk=%s: %s', run.pk, exc) + self.stderr.write(f'Run pk={run.pk}: projection failed ({exc}) -- skipping') + failed_count += 1 + continue + + if dry_run: + self.stdout.write( + f'Done (dry run). candidates: {len(candidates)}, ' + f'would_backfill: {would_backfill_count}, ' + f'skipped: {skipped_count}' + ) + else: + self.stdout.write( + f'Done. candidates: {len(candidates)}, ' + f'backfilled: {backfilled_count}, ' + f'skipped: {skipped_count}, ' + f'failed: {failed_count}' + ) + return diff --git a/solsys_code/management/commands/load_telescope_runs.py b/solsys_code/management/commands/load_telescope_runs.py index cf6c09b7..dcce5604 100644 --- a/solsys_code/management/commands/load_telescope_runs.py +++ b/solsys_code/management/commands/load_telescope_runs.py @@ -69,12 +69,15 @@ def _iter_run_nights(parsed: ParsedRun) -> list[date]: list[date]: evening dates for each night of the run. Raises: - ValueError: if day2 < day1 (cross-month ranges are not supported in - Phase 3), or if an ESO noon-to-noon range leaves no observing nights - after dropping its closing boundary (day2 <= day1). + ValueError: if day2 < day1 (a descending or malformed same-month day + range -- e.g. a typo like '20-5 July' -- that parse_run_line does + not reject upstream; genuine cross-month ranges are already + rejected at parse time by parse_run_line, PR-REVIEW-F2), or if an + ESO noon-to-noon range leaves no observing nights after dropping + its closing boundary (day2 <= day1). """ if parsed.day2 < parsed.day1: - raise ValueError(f'Cross-month run ranges not yet supported in Phase 3: {parsed!r}') + raise ValueError(f'Invalid or descending same-month day range (day2 < day1): {parsed!r}') n_nights = parsed.day2 - parsed.day1 + 1 if parsed.telescope in ESO_NOON_TO_NOON_SITES: # Tatoo's End date is the closing noon boundary of the last night, not an diff --git a/solsys_code/telescope_runs.py b/solsys_code/telescope_runs.py index 67b280e3..aa9de5ab 100644 --- a/solsys_code/telescope_runs.py +++ b/solsys_code/telescope_runs.py @@ -417,8 +417,10 @@ def parse_run_line(line: str) -> ParsedRun: Raises: ValueError: if line is empty, the telescope token does not resolve to exactly one SITES key (D-01), the status is unrecognized (D-06), - no date range can be found, or the trailing window token is present - but malformed. + no date range can be found, a genuine cross-month range is present + (PR-REVIEW-F2: not yet supported -- rejected at parse time instead + of being parsed into a ParsedRun the loader always rejects), or the + trailing window token is present but malformed. """ stripped = line.strip() if not stripped: @@ -426,32 +428,34 @@ def parse_run_line(line: str) -> ParsedRun: status, remainder = _resolve_status(stripped) - # Date range: try month-after-range ('Jul 8-12'), cross-month - # ('28 December-2 January'), then month-before-range ('9-13 July'). + # Date range: try month-after-range ('Jul 8-12') first, then check for a + # genuine cross-month range ('28 December-2 January') and reject it + # immediately (PR-REVIEW-F2: fail fast at parse time -- cross-month ranges + # are not yet supported, so there is no point building a ParsedRun the + # loader would always reject downstream), then fall back to + # month-before-range ('9-13 July'). match = _MONTH_AFTER_RANGE.search(remainder) if match: day1 = int(match.group('day1')) day2 = int(match.group('day2')) month = _MONTH_NAMES[match.group('month1').lower()] else: - match = _CROSS_MONTH_RANGE.search(remainder) - if match: - day1 = int(match.group('day1')) - day2 = int(match.group('day2')) - month = _MONTH_NAMES[match.group('month1').lower()] - else: - match = _MONTH_BEFORE_RANGE.search(remainder) - if not match: - raise ValueError(f'Could not find a date range (e.g. "9-13 July" or "Jul 8-12") in {line!r}') - day1 = int(match.group('day1')) - day2 = int(match.group('day2')) - month = _MONTH_NAMES[match.group('month1').lower()] - - # Year (PARSE-03): default to current year; roll over to next year if the - # run starts in December and ends in January (cross-year range). + cross_month_match = _CROSS_MONTH_RANGE.search(remainder) + if cross_month_match: + raise ValueError(f'Cross-month run ranges not yet supported: {line!r}') + match = _MONTH_BEFORE_RANGE.search(remainder) + if not match: + raise ValueError(f'Could not find a date range (e.g. "9-13 July" or "Jul 8-12") in {line!r}') + day1 = int(match.group('day1')) + day2 = int(match.group('day2')) + month = _MONTH_NAMES[match.group('month1').lower()] + + # Year (PARSE-03): default to current year. The previous December-to- + # January rollover here only ever served a genuine cross-month range, + # which now fails fast above; any remaining descending same-month range + # that reaches this point (e.g. a typo like '20-5 December') is always + # rejected downstream by _iter_run_nights, so no rollover is observable. year = date_cls.today().year - if month == 12 and day2 < day1: - year += 1 # Telescope (token 0) and instrument (token 1, possibly hyphenated). before_range = remainder[: match.start()] @@ -475,7 +479,10 @@ def parse_run_line(line: str) -> ParsedRun: window_tokens = after_range.split() start_window = end_window = None if len(window_tokens) == 1: - window_match = _PARTIAL_NIGHTS.search(window_tokens[0]) + # PR-REVIEW-F3: fullmatch (not search) so a token with surrounding garbage + # (e.g. 'xBoN-0626') is rejected rather than substring-matched into a + # plausible-but-wrong window. + window_match = _PARTIAL_NIGHTS.fullmatch(window_tokens[0]) if window_match: start_window = window_match.group(1) end_window = window_match.group(2) diff --git a/solsys_code/tests/test_backfill_range_calendar_events.py b/solsys_code/tests/test_backfill_range_calendar_events.py new file mode 100644 index 00000000..6e359213 --- /dev/null +++ b/solsys_code/tests/test_backfill_range_calendar_events.py @@ -0,0 +1,124 @@ +"""Tests for the backfill_range_calendar_events management command (D-07/FIX-08). + +Covers: a qualifying APPROVED, site-resolved, range-window CampaignRun with no existing +CAMPAIGN:{pk}* CalendarEvent gets one dip-corrected event per night via delegation to +campaign_views._project_calendar_event(); non-qualifying runs (single-night, TBD, +unresolved-site, PENDING_REVIEW) get none; a re-run is idempotent (no duplicates); +--dry-run writes nothing; and a per-candidate sun_event() ValueError is reported and +skipped, never aborting the whole backfill run. + +This module never fixtures an individual tom_targets.models.Target at all +(CampaignRun.target is nullable and left unset throughout), so CLAUDE.md's +non-sidereal-only target-factory convention doesn't arise here. +""" + +from datetime import date +from io import StringIO +from unittest.mock import patch + +from django.core.management import call_command +from django.db.models import Q +from django.test import TestCase +from tom_calendar.models import CalendarEvent +from tom_targets.models import TargetList + +from solsys_code.models import CampaignRun +from solsys_code.solsys_code_observatory.models import Observatory + + +class TestBackfillRangeCalendarEvents(TestCase): + """D-07/FIX-08: the one-off backfill command for already-APPROVED range-window runs.""" + + @classmethod + def setUpTestData(cls) -> None: + cls.campaign = TargetList.objects.create(name='3I/ATLAS') + # Tier-1-resolvable ground site so sun_event() succeeds deterministically without a + # live MPC call. + cls.ground_site = Observatory.objects.create( + obscode='F65', + name='Faulkes Telescope South', + short_name='FTS', + lat=-31.2727, + lon=149.0644, + altitude=1149.0, + timezone='Australia/Sydney', + observations_type=Observatory.OPTICAL_OBSTYPE, + ) + + def _make_approved_run(self, **overrides): + """Create an APPROVED CampaignRun; kwargs override the default 4-night ground window.""" + kwargs = { + 'campaign': self.campaign, + 'telescope_instrument': 'FTN/MuSCAT3', + 'site': self.ground_site, + 'window_start': date(2026, 8, 1), + 'window_end': date(2026, 8, 4), + 'approval_status': CampaignRun.ApprovalStatus.APPROVED, + } + kwargs.update(overrides) + return CampaignRun.objects.create(**kwargs) + + def _event_count(self, run): + return CalendarEvent.objects.filter( + Q(url=f'CAMPAIGN:{run.pk}') | Q(url__startswith=f'CAMPAIGN:{run.pk}:') + ).count() + + def test_backfill_projects_per_night_events_for_qualifying_range_run(self): + run = self._make_approved_run() + self.assertEqual(self._event_count(run), 0) + + call_command('backfill_range_calendar_events', stdout=StringIO()) + + self.assertEqual(self._event_count(run), 4) + self.assertTrue(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}:2026-08-01').exists()) + self.assertTrue(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}:2026-08-04').exists()) + + def test_backfill_is_idempotent_on_second_run(self): + run = self._make_approved_run() + call_command('backfill_range_calendar_events', stdout=StringIO()) + self.assertEqual(self._event_count(run), 4) + + call_command('backfill_range_calendar_events', stdout=StringIO()) + + self.assertEqual(self._event_count(run), 4) + + def test_backfill_skips_non_qualifying_runs(self): + single_night_run = self._make_approved_run(window_start=date(2026, 8, 10), window_end=date(2026, 8, 10)) + tbd_run = self._make_approved_run(window_start=None, window_end=None) + unresolved_site_run = self._make_approved_run( + site=None, window_start=date(2026, 8, 20), window_end=date(2026, 8, 23) + ) + pending_run = self._make_approved_run( + window_start=date(2026, 8, 25), + window_end=date(2026, 8, 28), + approval_status=CampaignRun.ApprovalStatus.PENDING_REVIEW, + ) + qualifying_run = self._make_approved_run() + + call_command('backfill_range_calendar_events', stdout=StringIO()) + + self.assertEqual(self._event_count(qualifying_run), 4) + self.assertEqual(self._event_count(single_night_run), 0) + self.assertEqual(self._event_count(tbd_run), 0) + self.assertEqual(self._event_count(unresolved_site_run), 0) + self.assertEqual(self._event_count(pending_run), 0) + + def test_backfill_dry_run_writes_nothing(self): + run = self._make_approved_run() + out = StringIO() + + call_command('backfill_range_calendar_events', '--dry-run', stdout=out) + + self.assertEqual(self._event_count(run), 0) + self.assertIn(str(run.pk), out.getvalue()) + self.assertIn('would', out.getvalue().lower()) + + def test_backfill_skips_and_continues_on_sun_event_valueerror(self): + run_a = self._make_approved_run() + run_b = self._make_approved_run(telescope_instrument='FTS/Spectral') + + with patch('solsys_code.campaign_views.sun_event', side_effect=ValueError('blank timezone')): + call_command('backfill_range_calendar_events', stdout=StringIO(), stderr=StringIO()) + + self.assertEqual(self._event_count(run_a), 0) + self.assertEqual(self._event_count(run_b), 0) diff --git a/solsys_code/tests/test_campaign_approval.py b/solsys_code/tests/test_campaign_approval.py index fd12117d..34b9796c 100644 --- a/solsys_code/tests/test_campaign_approval.py +++ b/solsys_code/tests/test_campaign_approval.py @@ -3,11 +3,12 @@ Covers: staff-only gating on both the approval-queue GET and the decision-endpoint POST (never a soft-filter -- a redirect, never 200-with-pending-content, per 16-RESEARCH.md Pitfall 7); the atomic conditional approve/reject transition and its proven double-approve no-op -(SUBMIT-03); the D-06 hybrid CAMPAIGN:{pk} CalendarEvent projection that fires only for a -single concrete night (window_start == window_end) with a resolved site -- a dip-corrected -sun_event() window for a ground site, a midnight-UTC placeholder for a space site -(CAL-01/CAL-02); no duplicate event and no ``modified`` churn on re-approve (CAL-03); and the -reject path (no event created). +(SUBMIT-03); CalendarEvent projection for a resolved-site, resolved-window run -- a single +night gets one dip-corrected/midnight-UTC event under the bare CAMPAIGN:{pk} key, a ground +range window gets one dip-corrected event per night under CAMPAIGN:{pk}:{date} keys, and a +satellite range window gets one whole-day-span event under the bare key (Phase 25 FIX-01..07); +no duplicate event and no ``modified`` churn on re-approve (CAL-03); and the reject path (no +event created). Uses ``TargetList.objects.create(...)`` for the campaign container and plain ``CampaignRun.objects.create(...)`` fixtures. This module never fixtures an individual @@ -22,6 +23,7 @@ import requests from django.contrib.auth.models import User from django.core.cache import cache +from django.db.models import Q from django.template.loader import render_to_string from django.test import TestCase, override_settings from django.urls import reverse @@ -298,11 +300,15 @@ def test_oversized_site_selection_is_flagged_with_no_network_call_or_fabrication class TestCalendarProjection(CampaignApprovalTestBase): - """D-06/CAL-01/CAL-02: approving a single-night run with a resolved site projects a - CAMPAIGN:{pk} event -- a dip-corrected sun_event() window for a ground site, a - midnight-UTC placeholder for a space site. A range, TBD run, missing - telescope_instrument, or a sun_event() ValueError all project nothing (the last of - these without reverting the already-committed approval). + """CAL-01/CAL-02/Phase 25 FIX-01..04: approving a resolved-site run with a resolved + window projects CalendarEvent(s) -- a single-night ground/space run gets one event under + the bare CAMPAIGN:{pk} key (dip-corrected sun_event() window for ground, midnight-UTC + placeholder for space); a ground range-window run gets one dip-corrected event per night + under CAMPAIGN:{pk}:{date} keys; a satellite range-window run still gets exactly one + whole-day-span event under the bare key. Only a TBD run, a run missing + telescope_instrument, or a sun_event() ValueError project nothing (the last of these + without reverting the already-committed approval; a mid-window ValueError leaves the + earlier nights' already-created events in place -- partial projection, no rollback). """ @classmethod @@ -346,12 +352,71 @@ def test_approve_single_night_space_run_creates_midnight_utc_placeholder_event(s self.assertEqual(event.start_time, datetime(2026, 8, 1, 0, 0, tzinfo=timezone.utc)) self.assertEqual(event.end_time, datetime(2026, 8, 1, 23, 59, tzinfo=timezone.utc)) - def test_approve_range_run_creates_no_calendar_event(self): + def test_approve_range_run_creates_one_event_per_night(self): + """FIX-01/FIX-02/FIX-03: a ground range-window run projects one dip-corrected + CalendarEvent per night, keyed CAMPAIGN:{pk}:{date}, with the first night's + start_time and the last night's end_time matching sun_event() for window_start/ + window_end respectively, and every event's title carrying the D-06 window suffix. + """ run = self._make_pending_run(window_start=date(2026, 8, 1), window_end=date(2026, 8, 15)) self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) run.refresh_from_db() self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) - self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + combined = CalendarEvent.objects.filter(Q(url=f'CAMPAIGN:{run.pk}') | Q(url__startswith=f'CAMPAIGN:{run.pk}:')) + self.assertEqual(combined.count(), 15) + + first_event = CalendarEvent.objects.get(url=f'CAMPAIGN:{run.pk}:2026-08-01') + expected_sunset, _ = sun_event(self.ground_site, date(2026, 8, 1), kind='sun') + self.assertEqual( + first_event.start_time, expected_sunset.to_datetime(timezone=timezone.utc).replace(microsecond=0) + ) + self.assertIn('(window 2026-08-01..2026-08-15)', first_event.title) + + last_event = CalendarEvent.objects.get(url=f'CAMPAIGN:{run.pk}:2026-08-15') + _, expected_sunrise = sun_event(self.ground_site, date(2026, 8, 15), kind='sun') + self.assertEqual( + last_event.end_time, expected_sunrise.to_datetime(timezone=timezone.utc).replace(microsecond=0) + ) + + def test_approve_range_run_space_site_creates_single_whole_day_span_event(self): + """FIX-04: a satellite range-window run still projects exactly one whole-day-span + event under the bare key (D-05 date-math unchanged), with the D-06 window suffix + applied to its title (Open Question 1 / Assumption A2, resolved uniformly).""" + space_site = Observatory.objects.create( + obscode='250', + name='Test Space Telescope', + short_name='TST', + observations_type=Observatory.SATELLITE_OBSTYPE, + ) + run = self._make_pending_run( + site_raw=space_site.obscode, window_start=date(2026, 8, 1), window_end=date(2026, 8, 15) + ) + self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + combined = CalendarEvent.objects.filter(Q(url=f'CAMPAIGN:{run.pk}') | Q(url__startswith=f'CAMPAIGN:{run.pk}:')) + self.assertEqual(combined.count(), 1) + event = CalendarEvent.objects.get(url=f'CAMPAIGN:{run.pk}') + self.assertEqual(event.start_time, datetime(2026, 8, 1, 0, 0, tzinfo=timezone.utc)) + self.assertEqual(event.end_time, datetime(2026, 8, 15, 23, 59, tzinfo=timezone.utc)) + self.assertIn('(window 2026-08-01..2026-08-15)', event.title) + + def test_approve_range_run_partial_projection_on_mid_window_sun_event_error(self): + """RESEARCH Open Question 2 / Assumption A3 lock: a sun_event() ValueError partway + through a range projects the earlier nights' events and leaves them in place -- + approve() swallows the raise, and there is no transaction.atomic() rollback.""" + real_sun_event = sun_event + + def _side_effect(site, night, kind='sun'): + if night == date(2026, 8, 3): + raise ValueError('no crossings') + return real_sun_event(site, night, kind=kind) + + run = self._make_pending_run(window_start=date(2026, 8, 1), window_end=date(2026, 8, 4)) + with patch('solsys_code.campaign_views.sun_event', side_effect=_side_effect): + self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) + run.refresh_from_db() + self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) + combined = CalendarEvent.objects.filter(Q(url=f'CAMPAIGN:{run.pk}') | Q(url__startswith=f'CAMPAIGN:{run.pk}:')) + self.assertEqual(combined.count(), 2) def test_approve_tbd_run_creates_no_calendar_event(self): run = self._make_pending_run(window_start=None, window_end=None) @@ -381,13 +446,14 @@ def test_sun_event_valueerror_skips_projection_without_reverting_approval(self): class TestRunStatusChange(CampaignApprovalTestBase): - """D-03/D-04/D-05: staff mark an APPROVED run cancelled or weathered from the Decided - table, and the linked CAMPAIGN:{pk} CalendarEvent (if one exists) updates in place with - a distinct terminal title prefix. A range/TBD/unresolved-site run that never had a - projected event is handled without crashing or fabricating one (RESEARCH Pitfall 1). A - non-APPROVED run, and a lost-update race between the guard read and the conditional - write (REVIEW finding #1), are both rejected/short-circuited server-side without a 500 - or a calendar mutation. + """D-03/D-04/D-05/Phase 25 FIX-05: staff mark an APPROVED run cancelled or weathered + from the Decided table, and every CalendarEvent belonging to it (the bare-key single + event, or all of a range run's per-night events) updates in place with a distinct + terminal title prefix, with the D-06 window suffix surviving the prefix transition. A + TBD/unresolved-site run that never had a projected event is handled without crashing or + fabricating one (RESEARCH Pitfall 1). A non-APPROVED run, and a lost-update race between + the guard read and the conditional write (REVIEW finding #1), are both rejected/ + short-circuited server-side without a 500 or a calendar mutation. """ @classmethod @@ -447,15 +513,23 @@ def test_mark_weather_failure_uses_distinct_weathered_prefix(self): self.assertFalse(event.title.startswith('[CANCELLED]')) self.assertIn('Run status: Weather/Technical Failure', event.description) - def test_mark_range_window_run_does_not_crash_and_creates_no_event(self): + def test_mark_range_window_run_updates_every_night_event(self): + """FIX-05: approving a 15-night range projects 15 events; marking it cancelled + updates every one of them in place, and the D-06 window suffix survives the + [CANCELLED] prefix transition (Pitfall 1).""" run = self._make_approved_single_night_run(window_start=date(2026, 8, 1), window_end=date(2026, 8, 15)) - self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + combined = CalendarEvent.objects.filter(Q(url=f'CAMPAIGN:{run.pk}') | Q(url__startswith=f'CAMPAIGN:{run.pk}:')) + self.assertEqual(combined.count(), 15) response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'mark_cancelled'}) self.assertEqual(response.status_code, 302) run.refresh_from_db() self.assertEqual(run.run_status, CampaignRun.RunStatus.CANCELLED) - self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + combined = CalendarEvent.objects.filter(Q(url=f'CAMPAIGN:{run.pk}') | Q(url__startswith=f'CAMPAIGN:{run.pk}:')) + self.assertEqual(combined.count(), 15) + for event in combined: + self.assertTrue(event.title.startswith('[CANCELLED] ')) + self.assertIn('(window 2026-08-01..2026-08-15)', event.title) def test_mark_status_on_non_approved_run_rejected(self): run = self._make_pending_run() # still PENDING_REVIEW -- never approved @@ -514,6 +588,27 @@ def test_mark_status_anonymous_or_non_staff_makes_no_change(self): run.refresh_from_db() self.assertEqual(run.run_status, CampaignRun.RunStatus.REQUESTED) + def test_mark_cancelled_survives_calendar_sync_failure(self): + """PR-REVIEW-F1: run_status is committed by the conditional `.update()` before the + calendar-sync loop runs, so a sync exception must never revert to a 500 -- it should + redirect (200 after follow) with the status change intact and a warning message. + """ + run = self._make_approved_single_night_run() + self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 1) + + with patch( + 'solsys_code.campaign_views.insert_or_create_calendar_event', + side_effect=Exception('simulated calendar sync failure'), + ): + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'mark_cancelled'}, follow=True + ) + self.assertEqual(response.status_code, 200) + run.refresh_from_db() + self.assertEqual(run.run_status, CampaignRun.RunStatus.CANCELLED) + messages_list = [str(m) for m in response.context['messages']] + self.assertTrue(any('could not be synced' in m for m in messages_list)) + class TestDecidedTableStatusActions(CampaignApprovalTestBase): """D-04 (Plan 02): the Decided table's Mark Cancelled/Mark Weathered action is gated by @@ -994,7 +1089,9 @@ def test_resolve_rejects_already_resolved_run(self): messages_list = [str(m) for m in response.context['messages']] self.assertIn('This run is not awaiting site resolution.', messages_list) - def test_resolve_range_tbd_run_clears_flag_with_no_calendar_event(self): + def test_resolve_range_run_projects_per_night_calendar_events(self): + """FIX-06: resolving a range run's site retroactively projects one event per night + and reports the 'added to the calendar' success message.""" run = self._make_needs_review_run(site_raw='F65', window_start=date(2026, 8, 1), window_end=date(2026, 8, 15)) response = self.client.post( reverse('campaigns:decide', kwargs={'pk': run.pk}), @@ -1005,7 +1102,27 @@ def test_resolve_range_tbd_run_clears_flag_with_no_calendar_event(self): run.refresh_from_db() self.assertEqual(run.site_id, self.ground_site.pk) self.assertFalse(run.site_needs_review) - self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + combined = CalendarEvent.objects.filter(Q(url=f'CAMPAIGN:{run.pk}') | Q(url__startswith=f'CAMPAIGN:{run.pk}:')) + self.assertEqual(combined.count(), 15) + messages_list = [str(m) for m in response.context['messages']] + self.assertIn('Site resolved — run added to the calendar.', messages_list) + + def test_resolve_tbd_run_clears_flag_with_no_calendar_event(self): + """Genuine TBD-resolve case (window_start/window_end both None): resolving the site + clears the review flag but projects zero events -- preserves the coverage the old + misnamed range test never actually exercised.""" + run = self._make_needs_review_run(site_raw='F65', window_start=None, window_end=None) + response = self.client.post( + reverse('campaigns:decide', kwargs={'pk': run.pk}), + {'action': 'resolve_site', 'site_selection': 'F65'}, + follow=True, + ) + self.assertEqual(response.status_code, 200) + run.refresh_from_db() + self.assertEqual(run.site_id, self.ground_site.pk) + self.assertFalse(run.site_needs_review) + combined = CalendarEvent.objects.filter(Q(url=f'CAMPAIGN:{run.pk}') | Q(url__startswith=f'CAMPAIGN:{run.pk}:')) + self.assertEqual(combined.count(), 0) messages_list = [str(m) for m in response.context['messages']] self.assertIn('Site resolved.', messages_list) @@ -2127,14 +2244,15 @@ def test_resolve_site_i11_resolves_gemini_south_ground_based(self): class TestGeminiFtScenario(CampaignApprovalTestBase): - """D-06/D-07: the real Gemini Fast-Turnaround GS-2026A-FT-115 informational run flows - through the SAME approve -> mark-status mechanism as any Magellan run, with no - special-casing. Its window is a 4-day range (2026-07-13..2026-07-16), so approving it - projects NO ``CAMPAIGN:{pk}`` ``CalendarEvent`` (range-window skip-by-design); marking - it weathered, then cancelled, must set ``run_status`` only and never crash or - fabricate an event (RESEARCH Pitfall 1, T-23-07). This scenario creates no new - production code -- it exercises Plan 02's already-built ``_set_run_status()`` - end-to-end against the real D-06 seed values. + """D-06/D-07/Phase 25 FIX-02/FIX-05: the real Gemini Fast-Turnaround GS-2026A-FT-115 + informational run flows through the SAME approve -> mark-status mechanism as any + Magellan run, with no special-casing. Its window is a 4-day range + (2026-07-13..2026-07-16), so approving it projects 4 per-night ``CalendarEvent``s; + marking it weathered, then cancelled, sets ``run_status`` and updates every one of + those 4 events in place with the matching terminal prefix, preserving the D-06 window + suffix (RESEARCH Pitfall 1, T-23-07). This scenario exercises the real, rewritten + ``_project_calendar_event()``/``_set_run_status()`` end-to-end against the real D-06 + seed values. """ @classmethod @@ -2159,7 +2277,7 @@ def setUpTestData(cls) -> None: def setUp(self): self.client.login(username='staffcoordinator', password='pw') - def test_gemini_ft115_range_window_flows_through_same_mechanism_no_event_fabricated(self): + def test_gemini_ft115_range_window_projects_per_night_events(self): run = self._make_pending_run( campaign=self.didymos_campaign, telescope_instrument='Gemini-South GMOS-S', @@ -2173,33 +2291,44 @@ def test_gemini_ft115_range_window_flows_through_same_mechanism_no_event_fabrica target=None, ) - # (a) Approve: because the window is a 4-day range, no CalendarEvent is projected - # (range-window projection is skipped by design). + def _combined(): + return CalendarEvent.objects.filter(Q(url=f'CAMPAIGN:{run.pk}') | Q(url__startswith=f'CAMPAIGN:{run.pk}:')) + + # (a) Approve: the 4-day window projects 4 per-night CalendarEvents. response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'approve'}) self.assertEqual(response.status_code, 302) run.refresh_from_db() self.assertEqual(run.approval_status, CampaignRun.ApprovalStatus.APPROVED) self.assertEqual(run.site_id, self.gemini_south.pk) - self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + self.assertEqual(_combined().count(), 4) # (b)/(c)/(d) mark_weather_failure: normal redirect (no 500/IntegrityError), - # run_status set to WEATHER_TECH_FAILURE, still no CalendarEvent fabricated. + # run_status set to WEATHER_TECH_FAILURE, all 4 events updated with the + # [WEATHERED] prefix and the window suffix preserved. response = self.client.post( reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'mark_weather_failure'} ) self.assertEqual(response.status_code, 302) run.refresh_from_db() self.assertEqual(run.run_status, CampaignRun.RunStatus.WEATHER_TECH_FAILURE) - self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + combined = _combined() + self.assertEqual(combined.count(), 4) + for event in combined: + self.assertTrue(event.title.startswith('[WEATHERED] ')) + self.assertIn('(window 2026-07-13..2026-07-16)', event.title) # A follow-up mark_cancelled is a REAL transition (WEATHER_TECH_FAILURE -> # CANCELLED are two distinct RunStatus values, not an idempotent no-op -- - # REVIEW finding, Codex MEDIUM); still no CalendarEvent fabricated. + # REVIEW finding, Codex MEDIUM); all 4 events updated with [CANCELLED]. response = self.client.post(reverse('campaigns:decide', kwargs={'pk': run.pk}), {'action': 'mark_cancelled'}) self.assertEqual(response.status_code, 302) run.refresh_from_db() self.assertEqual(run.run_status, CampaignRun.RunStatus.CANCELLED) - self.assertEqual(CalendarEvent.objects.filter(url=f'CAMPAIGN:{run.pk}').count(), 0) + combined = _combined() + self.assertEqual(combined.count(), 4) + for event in combined: + self.assertTrue(event.title.startswith('[CANCELLED] ')) + self.assertIn('(window 2026-07-13..2026-07-16)', event.title) # Source assertion anchor (exact D-06 seed values, target left unset): self.assertEqual(run.telescope_instrument, 'Gemini-South GMOS-S') diff --git a/solsys_code/tests/test_load_telescope_runs.py b/solsys_code/tests/test_load_telescope_runs.py index c06471a8..24a48316 100644 --- a/solsys_code/tests/test_load_telescope_runs.py +++ b/solsys_code/tests/test_load_telescope_runs.py @@ -308,6 +308,26 @@ def test_unparseable_line_logged_and_skipped(self): self.assertIn('2', err, 'Expected line number in stderr error message') self.assertIn('Magellan IMACS 13-19 July (proposed)', err) + def test_cross_month_line_logged_and_skipped(self): + """PR-REVIEW-F2: a genuine cross-month run line is rejected at parse time (fail-fast) + and logged to stderr with its line number, not crashed on; the valid line still + processes into its CalendarEvents.""" + path, tmpdir_ctx = self._write_schedule_file( + [ + 'NTT EFOSC2 28 December-2 January', + 'NTT EFOSC2 allocation 9-13 July', + ] + ) + with tmpdir_ctx: + stderr_buf = io.StringIO() + call_command('load_telescope_runs', path, stdout=io.StringIO(), stderr=stderr_buf) + # The cross-month line should produce no events; the valid NTT line should still + # create 4 events (ESO noon-to-noon: nights 9-12 July). + self.assertEqual(CalendarEvent.objects.count(), 4) + err = stderr_buf.getvalue() + self.assertIn('1', err, 'Expected line number in stderr error message') + self.assertIn('NTT EFOSC2 28 December-2 January', err) + def test_partial_night_bon_to_hhmm_sets_end_time(self): """INGEST-WIN-01: a BoN-HHMM window line sets end_time to HHMM UTC on d+1 morning.""" path, tmpdir_ctx = self._write_schedule_file(['Magellan-Clay Lightspeed 18-20 July BoN-0626']) diff --git a/solsys_code/tests/test_telescope_runs.py b/solsys_code/tests/test_telescope_runs.py index f678184b..c7d1279d 100644 --- a/solsys_code/tests/test_telescope_runs.py +++ b/solsys_code/tests/test_telescope_runs.py @@ -358,13 +358,19 @@ def test_parse_run_line_no_year_defaults_to_current_year(self): result = parse_run_line('FTS Spectral confirmed 5-7 Jan') self.assertEqual(result.year, date.today().year) - def test_parse_run_line_december_january_rolls_over_year(self): - """ROADMAP SC4 / PARSE-03: a late-December-start range rolls year to current year + 1.""" - result = parse_run_line('NTT EFOSC2 28 December-2 January') - self.assertEqual(result.year, date.today().year + 1) - self.assertEqual(result.month, 12) - self.assertEqual(result.day1, 28) - self.assertEqual(result.day2, 2) + def test_parse_run_line_cross_month_range_raises(self): + """PR-REVIEW-F2: a genuine cross-month range now fails fast at parse time instead of + being parsed into a ParsedRun the loader always rejects downstream.""" + with self.assertRaises(ValueError) as ctx: + parse_run_line('NTT EFOSC2 28 December-2 January') + self.assertIn('Cross-month', str(ctx.exception)) + + def test_parse_run_line_partial_night_token_with_garbage_prefix_raises(self): + """PR-REVIEW-F3: fullmatch anchoring rejects a partial-night token with surrounding + garbage (e.g. 'xBoN-0626') instead of substring-matching the well-formed 'BoN-0626' + inside it.""" + with self.assertRaises(ValueError): + parse_run_line('Magellan-Clay Lightspeed 18-20 July xBoN-0626') def test_parse_run_line_no_status_defaults_to_allocation(self): """D-05: a run line with no status defaults to status='allocation'.""" From 252c2b0eb0ad6b6639d9aae9e36792b10241f348 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Sun, 19 Jul 2026 18:17:35 -0700 Subject: [PATCH 3/3] Add backfill_lco_observation_records management command Backfills FOMO ObservationRecords for LCO RequestGroups submitted directly at the LCO observing portal (outside the TOM), by proposal and RequestGroup name prefix, matching each request's target against a chosen campaign TargetList's members. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../backfill_lco_observation_records.py | 243 ++++++++++++++++++ .../test_backfill_lco_observation_records.py | 173 +++++++++++++ 2 files changed, 416 insertions(+) create mode 100644 solsys_code/management/commands/backfill_lco_observation_records.py create mode 100644 solsys_code/tests/test_backfill_lco_observation_records.py diff --git a/solsys_code/management/commands/backfill_lco_observation_records.py b/solsys_code/management/commands/backfill_lco_observation_records.py new file mode 100644 index 00000000..27a5d625 --- /dev/null +++ b/solsys_code/management/commands/backfill_lco_observation_records.py @@ -0,0 +1,243 @@ +from typing import Any +from urllib.parse import urlencode, urljoin + +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand, CommandError, CommandParser +from tom_observations.facilities.lco import LCOFacility +from tom_observations.facilities.ocs import make_request +from tom_observations.models import ObservationRecord +from tom_targets.models import TargetList + + +def _matching_request_groups(facility: LCOFacility, proposal: str, name_prefix: str): + """Page through GET /api/requestgroups/ for a proposal, yielding name-prefix matches. + + The 'proposal' filter is an exact match; 'name' is icontains server-side, so it's + passed as a pre-filter to cut payload size, then re-checked client-side with a real + str.startswith() since icontains would also match the prefix appearing mid-string. + + Args: + facility: an LCOFacility instance (for portal_url/api_key settings and headers). + proposal: LCO proposal code, exact match. + name_prefix: only RequestGroups whose 'name' starts with this string are yielded. + + Yields: + dict: each matching RequestGroup object (with its nested 'requests' list). + """ + query = urlencode({'proposal': proposal, 'name': name_prefix, 'limit': 100}) + url = urljoin(facility.facility_settings.get_setting('portal_url'), f'/api/requestgroups/?{query}') + while url: + response = make_request('GET', url, headers=facility._portal_headers()) + payload = response.json() + for request_group in payload.get('results', []): + if request_group.get('name', '').startswith(name_prefix): + yield request_group + url = payload.get('next') + + +def _request_target_name(request: dict[str, Any]) -> str | None: + """Return the target name of a request's first configuration that has one.""" + for configuration in request.get('configurations', []): + target = configuration.get('target') or {} + if target.get('name'): + return target['name'] + return None + + +def _build_parameters(request_group: dict[str, Any], request: dict[str, Any]) -> dict[str, Any] | None: + """Build a minimal flat ObservationRecord.parameters dict for a backfilled request. + + Matches the legacy single-config flat shape ('proposal', 'start', 'end', + 'instrument_type') that solsys_code.calendar_utils._extract_instrument() falls back + to when no c_N_*-prefixed multi-configuration keys are present -- deliberately not + replicating the full submission-form c_N_* shape, since the RequestGroup API's + 'configurations' entries don't map 1:1 onto it. + + Args: + request_group: the parent RequestGroup object (for 'proposal'). + request: a single request from request_group['requests'] (for 'windows' and the + first configuration with an 'instrument_type'). + + Returns: + dict[str, Any] | None: the parameters dict, or None if the request has no + configuration with a usable instrument_type. + """ + for configuration in request.get('configurations', []): + instrument_type = configuration.get('instrument_type') + if not instrument_type: + continue + parameters = { + 'proposal': request_group.get('proposal'), + 'instrument_type': instrument_type, + } + windows = request.get('windows') or [] + if windows: + if windows[0].get('start'): + parameters['start'] = windows[0]['start'] + if windows[0].get('end'): + parameters['end'] = windows[0]['end'] + return parameters + return None + + +class Command(BaseCommand): + """Backfill ObservationRecords for LCO RequestGroups submitted outside the TOM. + + Queries the LCO Observation Portal's 'Get All RequestGroups' API + (GET /api/requestgroups/) for a proposal, keeps only RequestGroups whose name + starts with --name-prefix, and creates one ObservationRecord per child request + (mirroring LCORedirectFacility.request_id_to_group's per-request granularity) -- + skipping any request that already has an ObservationRecord (facility='LCO', + observation_id=). + + Each request's target is matched by name against the Targets already belonging to + a chosen campaign (a tom_targets.TargetList) -- a request whose target name isn't a + member of that campaign is skipped and logged, never guessed at. + """ + + help = 'Backfill ObservationRecords from LCO RequestGroups submitted directly at the LCO portal' + + def add_arguments(self, parser: CommandParser) -> None: + """Parse command line arguments.""" + parser.add_argument( + '--proposal', + required=True, + help='LCO proposal code to filter RequestGroups by (exact match).', + ) + parser.add_argument( + '--name-prefix', + required=True, + help="Only backfill RequestGroups whose 'name' starts with this prefix.", + ) + parser.add_argument( + '--campaign', + required=False, + help=( + 'Name of the campaign (tom_targets.TargetList) to match request targets against. ' + 'If omitted, you will be prompted to choose one interactively.' + ), + ) + parser.add_argument( + '--username', + required=False, + help='Attribute created ObservationRecords to this username (default: unattributed).', + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='Report what would be created without writing any ObservationRecord rows.', + ) + + def handle(self, *args: Any, **options: Any) -> str | None: + """Fetch matching RequestGroups and create ObservationRecords for their requests. + + Returns: + str | None: a one-line summary of created/skipped counts. + """ + proposal = options['proposal'] + name_prefix = options['name_prefix'] + dry_run = options['dry_run'] + + user = None + if options.get('username'): + try: + user = get_user_model().objects.get(username=options['username']) + except get_user_model().DoesNotExist as exc: + raise CommandError(f'Invalid username: {options["username"]!r}') from exc + + campaign = self._resolve_campaign(options.get('campaign')) + targets_by_name = {target.name: target for target in campaign.targets.all()} + if not targets_by_name: + raise CommandError(f'Campaign {campaign.name!r} has no targets to match requests against.') + + facility = LCOFacility() + facility.set_user(user) + + created = 0 + skipped_existing = 0 + skipped_unmatched_target = 0 + skipped_no_config = 0 + + for request_group in _matching_request_groups(facility, proposal, name_prefix): + for request in request_group.get('requests', []): + observation_id = str(request['id']) + if ObservationRecord.objects.filter(facility=facility.name, observation_id=observation_id).exists(): + skipped_existing += 1 + continue + + target_name = _request_target_name(request) + target = targets_by_name.get(target_name) + if target is None: + self.stderr.write( + f'Skipping request {observation_id} (group {request_group.get("name")!r}): ' + f'target {target_name!r} is not a member of campaign {campaign.name!r}.' + ) + skipped_unmatched_target += 1 + continue + + parameters = _build_parameters(request_group, request) + if parameters is None: + self.stderr.write( + f'Skipping request {observation_id}: no configuration with an instrument_type found.' + ) + skipped_no_config += 1 + continue + + if dry_run: + self.stdout.write( + f'Would create ObservationRecord: target={target.name!r}, observation_id={observation_id}, ' + f'status={request.get("state", "")!r}' + ) + else: + ObservationRecord.objects.create( + target=target, + user=user, + facility=facility.name, + observation_id=observation_id, + status=request.get('state', ''), + parameters=parameters, + ) + created += 1 + + summary = ( + f'{"Would create" if dry_run else "Created"}: {created}, already existed: {skipped_existing}, ' + f'unmatched target: {skipped_unmatched_target}, no usable configuration: {skipped_no_config}' + ) + self.stdout.write(summary) + return summary + + def _resolve_campaign(self, campaign_name: str | None) -> TargetList: + """Resolve --campaign to a TargetList, prompting interactively if not given. + + Args: + campaign_name: the --campaign value, or None to prompt. + + Returns: + TargetList: the resolved campaign. + + Raises: + CommandError: no TargetList by that name exists, more than one does, no + TargetLists exist at all, or an interactive selection was invalid. + """ + if campaign_name: + matches = TargetList.objects.filter(name=campaign_name) + if not matches.exists(): + raise CommandError(f'No campaign (TargetList) named {campaign_name!r} found.') + if matches.count() > 1: + raise CommandError(f'Multiple campaigns (TargetLists) named {campaign_name!r} found.') + return matches.first() + + target_lists = list(TargetList.objects.order_by('name')) + if not target_lists: + raise CommandError('No campaigns (TargetLists) exist to select from.') + self.stdout.write('Available campaigns:') + for index, target_list in enumerate(target_lists, start=1): + self.stdout.write(f' {index}. {target_list.name} ({target_list.targets.count()} targets)') + choice = input('Select a campaign by number: ').strip() + try: + selected_index = int(choice) - 1 + if selected_index < 0: + raise ValueError + return target_lists[selected_index] + except (ValueError, IndexError) as exc: + raise CommandError(f'Invalid selection: {choice!r}') from exc diff --git a/solsys_code/tests/test_backfill_lco_observation_records.py b/solsys_code/tests/test_backfill_lco_observation_records.py new file mode 100644 index 00000000..6616b13f --- /dev/null +++ b/solsys_code/tests/test_backfill_lco_observation_records.py @@ -0,0 +1,173 @@ +from unittest.mock import MagicMock, patch + +from django.core.management import CommandError, call_command +from django.test import TestCase +from tom_observations.models import ObservationRecord +from tom_targets.models import TargetList +from tom_targets.tests.factories import NonSiderealTargetFactory + + +def _configuration(instrument_type='1M0-SCICAM-SINISTRO', target_name='Didymos'): + return { + 'type': 'EXPOSE', + 'instrument_type': instrument_type, + 'instrument_configs': [{'exposure_time': 30.0, 'exposure_count': 1}], + 'target': {'name': target_name, 'type': 'ORBITAL_ELEMENTS'}, + } + + +def _request(request_id, target_name='Didymos', state='COMPLETED'): + return { + 'id': request_id, + 'state': state, + 'windows': [{'start': '2026-07-01T00:00:00', 'end': '2026-07-02T00:00:00'}], + 'configurations': [_configuration(target_name=target_name)], + } + + +def _request_group(group_id, name, proposal='LCO2026A-003', requests=None): + return { + 'id': group_id, + 'name': name, + 'proposal': proposal, + 'state': 'COMPLETED', + 'requests': requests if requests is not None else [_request(group_id * 10)], + } + + +def _page_response(results, next_url=None): + response = MagicMock() + response.json.return_value = {'count': len(results), 'next': next_url, 'previous': None, 'results': results} + return response + + +class TestBackfillLcoObservationRecords(TestCase): + @classmethod + def setUpTestData(cls): + cls.target = NonSiderealTargetFactory.create(name='Didymos') + cls.campaign = TargetList.objects.create(name='Didymos 2026 Campaign') + cls.campaign.targets.add(cls.target) + + @patch('solsys_code.management.commands.backfill_lco_observation_records.make_request') + def test_creates_record_for_matching_group_and_target(self, mock_make_request): + mock_make_request.return_value = _page_response([_request_group(1, 'Didymos 2026 - ELP')]) + + call_command( + 'backfill_lco_observation_records', + '--proposal=LCO2026A-003', + '--name-prefix=Didymos', + '--campaign=Didymos 2026 Campaign', + ) + + record = ObservationRecord.objects.get(facility='LCO', observation_id='10') + self.assertEqual(record.target, self.target) + self.assertEqual(record.status, 'COMPLETED') + self.assertEqual(record.parameters['proposal'], 'LCO2026A-003') + self.assertEqual(record.parameters['instrument_type'], '1M0-SCICAM-SINISTRO') + self.assertEqual(record.parameters['start'], '2026-07-01T00:00:00') + + @patch('solsys_code.management.commands.backfill_lco_observation_records.make_request') + def test_name_prefix_is_rechecked_client_side(self, mock_make_request): + # Server-side 'name' filter is icontains, so a group containing but not + # starting with the prefix could come back from the API -- must be excluded. + mock_make_request.return_value = _page_response([_request_group(1, 'Not a Didymos run')]) + + call_command( + 'backfill_lco_observation_records', + '--proposal=LCO2026A-003', + '--name-prefix=Didymos', + '--campaign=Didymos 2026 Campaign', + ) + + self.assertFalse(ObservationRecord.objects.exists()) + + @patch('solsys_code.management.commands.backfill_lco_observation_records.make_request') + def test_skips_request_with_existing_observation_record(self, mock_make_request): + ObservationRecord.objects.create( + target=self.target, facility='LCO', observation_id='10', status='COMPLETED', parameters={} + ) + mock_make_request.return_value = _page_response([_request_group(1, 'Didymos 2026 - ELP')]) + + call_command( + 'backfill_lco_observation_records', + '--proposal=LCO2026A-003', + '--name-prefix=Didymos', + '--campaign=Didymos 2026 Campaign', + ) + + self.assertEqual(ObservationRecord.objects.filter(facility='LCO', observation_id='10').count(), 1) + + @patch('solsys_code.management.commands.backfill_lco_observation_records.make_request') + def test_skips_request_whose_target_is_not_a_campaign_member(self, mock_make_request): + mock_make_request.return_value = _page_response( + [_request_group(1, 'Didymos 2026 - ELP', requests=[_request(10, target_name='Some Other Object')])] + ) + + call_command( + 'backfill_lco_observation_records', + '--proposal=LCO2026A-003', + '--name-prefix=Didymos', + '--campaign=Didymos 2026 Campaign', + ) + + self.assertFalse(ObservationRecord.objects.exists()) + + @patch('solsys_code.management.commands.backfill_lco_observation_records.make_request') + def test_dry_run_creates_nothing(self, mock_make_request): + mock_make_request.return_value = _page_response([_request_group(1, 'Didymos 2026 - ELP')]) + + call_command( + 'backfill_lco_observation_records', + '--proposal=LCO2026A-003', + '--name-prefix=Didymos', + '--campaign=Didymos 2026 Campaign', + '--dry-run', + ) + + self.assertFalse(ObservationRecord.objects.exists()) + + @patch('solsys_code.management.commands.backfill_lco_observation_records.make_request') + def test_follows_pagination(self, mock_make_request): + mock_make_request.side_effect = [ + _page_response([_request_group(1, 'Didymos 2026 - ELP')], next_url='https://observe.lco.global/next'), + _page_response([_request_group(2, 'Didymos 2026 - LSC', requests=[_request(20)])]), + ] + + call_command( + 'backfill_lco_observation_records', + '--proposal=LCO2026A-003', + '--name-prefix=Didymos', + '--campaign=Didymos 2026 Campaign', + ) + + self.assertEqual(mock_make_request.call_count, 2) + self.assertEqual(ObservationRecord.objects.filter(facility='LCO').count(), 2) + + def test_unknown_campaign_raises_command_error(self): + with self.assertRaises(CommandError): + call_command( + 'backfill_lco_observation_records', + '--proposal=LCO2026A-003', + '--name-prefix=Didymos', + '--campaign=Not A Real Campaign', + ) + + def test_campaign_with_no_targets_raises_command_error(self): + TargetList.objects.create(name='Empty Campaign') + with self.assertRaises(CommandError): + call_command( + 'backfill_lco_observation_records', + '--proposal=LCO2026A-003', + '--name-prefix=Didymos', + '--campaign=Empty Campaign', + ) + + def test_unknown_username_raises_command_error(self): + with self.assertRaises(CommandError): + call_command( + 'backfill_lco_observation_records', + '--proposal=LCO2026A-003', + '--name-prefix=Didymos', + '--campaign=Didymos 2026 Campaign', + '--username=nonexistent-user', + )