SD-149 preflight validation

dj-be/pyproject.toml

@@ -22,7 +22,8 @@ Authlib = "1.5.2"
 pandas = "1.5.3"
 python-docx = "1.1.2"
 requests = "2.32.4"
-pyxform = "1.12.2"
+pyxform = "4.5.0"
+lark = "1.3.1"
 django-admin-sortable2 = "2.2.2"
 django-rq = "2.10.3"
 django-nested-admin = "3.4.1"
@@ -49,7 +50,7 @@ idna = "3.7"
 setuptools = "78.1.1"
 zipp = "3.19.1"
 rq = "1.16.2"
-openpyxl = "3.0.9"
+openpyxl = "3.1.5"
 numpy = "1.25.2"
 
 [tool.poetry.group.dev.dependencies]

dj-be/survey_designer/apps/change_requests/tests/test_views.py

@@ -16,7 +16,6 @@
     SubmoduleRequiredGroup,
 )
 from openpyxl import load_workbook
-from openpyxl.writer.excel import save_virtual_workbook
 from organization.models import Organization
 from questions.const import QuestionType
 from questions.models import (
@@ -29,6 +28,7 @@
     Suffix,
 )
 from questions.services import QuestionsExport
+from questions.services.workbook import save_virtual_workbook
 
 
 def test_submit_change_request_view_get(

dj-be/survey_designer/apps/modules/tests/test_organization_scope.py

@@ -130,7 +130,8 @@ def test_assigned_admin_can_queue_generation_for_foreign_selected_scope(
 
 
 @pytest.mark.parametrize(
-    "endpoint", ["/api/generate/", "/api/preview/", "/api/upload/"]
+    "endpoint",
+    ["/api/generate/", "/api/preview/", "/api/upload/", "/api/validate/"],
 )
 def test_out_of_scope_content_fails_before_generation_or_external_side_effects(
     mocker,

dj-be/survey_designer/apps/modules/tests/test_views.py

@@ -9,6 +9,7 @@
 from django.core.cache import cache
 from django.core.files.base import ContentFile
 from django.db import connection
+from django.test import override_settings
 from django.test.utils import CaptureQueriesContext
 from django_rq import get_queue
 from documents.models import Document
@@ -20,6 +21,7 @@
     SubmoduleRequiredGroup,
 )
 from modules.views import generate_docx
+from pyxform.errors import PyXFormError
 from questions.const import QuestionType
 from questions.models import (
     RootQuestion,
@@ -62,6 +64,20 @@ def generate(self):
     return StubXLSForm(external_files)
 
 
+def mock_successful_xml_conversion(mocker, xml=None):
+    conversion = mocker.Mock()
+    conversion.run.return_value = xml or (
+        '<h:html xmlns:h="http://www.w3.org/1999/xhtml" '
+        'xmlns:xf="http://www.w3.org/2002/xforms">'
+        "<h:head><xf:model><xf:instance><data/></xf:instance>"
+        "</xf:model></h:head><h:body/></h:html>"
+    )
+    conversion.warnings = []
+    conversion.errors = []
+    mocker.patch("modules.views.XMLConversion", return_value=conversion)
+    return conversion
+
+
 @pytest.fixture(autouse=True)
 def selected_organization_header(
     logged_admin_client, api_client_authenticated_admin, organization_1
@@ -577,6 +593,92 @@ def test_preview_xls_form(
     assert Survey.objects.count() == 1
 
 
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    ("conversion_error", "expected_status", "expected_code"),
+    [
+        (
+            PyXFormError("invalid XLSForm"),
+            status.HTTP_400_BAD_REQUEST,
+            "PYXFORM_CONVERSION_ERROR",
+        ),
+        (
+            RuntimeError("converter crashed"),
+            status.HTTP_503_SERVICE_UNAVAILABLE,
+            "VALIDATOR_UNAVAILABLE",
+        ),
+        (
+            TimeoutError("converter timed out"),
+            status.HTTP_503_SERVICE_UNAVAILABLE,
+            "VALIDATOR_UNAVAILABLE",
+        ),
+    ],
+)
+def test_converter_failures_keep_input_and_infrastructure_statuses(
+    mocker,
+    api_client_authenticated_admin,
+    submodule_1,
+    conversion_error,
+    expected_status,
+    expected_code,
+):
+    mocker.patch(
+        "questions.services.xml_conversion.xls2xform.convert",
+        side_effect=conversion_error,
+    )
+    payload = {
+        "name": "Converter failure survey",
+        "submodules": [submodule_1.id],
+        "submodules_order": [submodule_1.id],
+        "sub_questions": [],
+        "languages": ["en"],
+    }
+
+    response = api_client_authenticated_admin.post(
+        "/api/validate/", payload, format="json"
+    )
+
+    assert response.status_code == expected_status
+    assert response.json()["errors"][0]["code"] == expected_code
+
+
+@pytest.mark.django_db
+@override_settings(CORS_ALLOWED_ORIGINS=["http://localhost:3000"])
+def test_download_exposes_validation_warning_headers_to_frontend(
+    mocker, api_client_authenticated_admin, submodule_1
+):
+    conversion = mock_successful_xml_conversion(mocker)
+    conversion.warnings = ["non-blocking warning"]
+    payload = {
+        "name": "Warning survey",
+        "submodules": [submodule_1.id],
+        "submodules_order": [submodule_1.id],
+        "sub_questions": [],
+        "languages": ["en"],
+    }
+
+    response = api_client_authenticated_admin.post(
+        "/api/generate/",
+        payload,
+        format="json",
+        HTTP_ORIGIN="http://localhost:3000",
+    )
+
+    assert response.status_code == status.HTTP_200_OK
+    warnings = json.loads(response["X-Survey-Validation-Warnings"])
+    assert warnings[0]["code"] == "PYXFORM_WARNING"
+    assert response["Access-Control-Allow-Origin"] == "http://localhost:3000"
+    exposed_headers = {
+        header.strip().lower()
+        for header in response["Access-Control-Expose-Headers"].split(",")
+    }
+    assert {
+        "x-survey-validation-warnings",
+        "x-validation-warnings",
+        "x-survey-artifact-hash",
+    }.issubset(exposed_headers)
+
+
 @pytest.mark.django_db
 def test_preview_xls_form_with_external_media(
     mocker,
@@ -591,13 +693,9 @@ def test_preview_xls_form_with_external_media(
         "fruits.csv": ContentFile(csv_content, name="fruits.csv")
     }
 
-    mocker.patch("modules.views.get_xlsx_from_request", return_value=stub_form)
+    mocker.patch("modules.views.get_xlsx_from_data", return_value=stub_form)
 
-    xml_conversion = mocker.Mock()
-    xml_conversion.run.return_value = "<data/>"
-    xml_conversion.warnings = []
-    xml_conversion.errors = []
-    mocker.patch("modules.views.XMLConversion", return_value=xml_conversion)
+    mock_successful_xml_conversion(mocker)
 
     saved_paths = []
 
@@ -670,7 +768,7 @@ def test_preview_xls_form_rewrites_external_file_links(
         "logo.png": ContentFile(img_content, name="logo.png"),
     }
 
-    mocker.patch("modules.views.get_xlsx_from_request", return_value=stub_form)
+    mocker.patch("modules.views.get_xlsx_from_data", return_value=stub_form)
 
     xml_payload = (
         '<h:html xmlns:h="http://www.w3.org/1999/xhtml" '
@@ -679,11 +777,7 @@ def test_preview_xls_form_rewrites_external_file_links(
         '<h:body><h:img src="jr://images/logo.png"/></h:body>'
         "</h:html>"
     )
-    xml_conversion = mocker.Mock()
-    xml_conversion.run.return_value = xml_payload
-    xml_conversion.warnings = []
-    xml_conversion.errors = []
-    mocker.patch("modules.views.XMLConversion", return_value=xml_conversion)
+    mock_successful_xml_conversion(mocker, xml_payload)
 
     saved_contents = {}
 
@@ -896,6 +990,7 @@ def test_upload_xls_form_moda_uploads_metadata(
     fake_file = FakeFieldFile("fruits.csv", b"name,color\nbanana,yellow\n")
     stub_form = build_stub_xls_form({"fruits.csv": fake_file})
     mocker.patch("modules.views.get_xlsx_from_data", return_value=stub_form)
+    mock_successful_xml_conversion(mocker)
 
     upload_response = mocker.Mock()
     upload_response.ok = True
@@ -966,6 +1061,7 @@ def test_upload_xls_form_moda_without_attachments(
     site = moda_api_key.site
     stub_form = build_stub_xls_form({})
     mocker.patch("modules.views.get_xlsx_from_data", return_value=stub_form)
+    mock_successful_xml_conversion(mocker)
 
     upload_response = mocker.Mock()
     upload_response.ok = True
@@ -1009,6 +1105,7 @@ def test_upload_xls_form_moda_metadata_failure(
 ):
     stub_form = build_stub_xls_form({"fruits.csv": FakeFieldFile("fruits.csv")})
     mocker.patch("modules.views.get_xlsx_from_data", return_value=stub_form)
+    mock_successful_xml_conversion(mocker)
 
     upload_response = mocker.Mock()
     upload_response.ok = True

dj-be/survey_designer/apps/modules/urls.py

@@ -10,6 +10,7 @@
     SubmodulesOrderValidationView,
     SubmoduleViewSet,
     UploadXLSForm,
+    ValidateXLSForm,
 )
 
 router = routers.SimpleRouter()
@@ -25,6 +26,7 @@
         path("generate-doc/", GenerateDocForm.as_view(), name="generate_doc_form"),
         path("upload/", UploadXLSForm.as_view(), name="upload_xls_form"),
         path("preview/", PreviewXLSForm.as_view(), name="preview_xls_form"),
+        path("validate/", ValidateXLSForm.as_view(), name="validate_xls_form"),
         path(
             "order-validation/",
             SubmodulesOrderValidationView.as_view(),